diff --git a/_sources/generated/demos/Main_Demo.ipynb.txt b/_sources/generated/demos/Main_Demo.ipynb.txt
index c0fed32d9..41853de67 100644
--- a/_sources/generated/demos/Main_Demo.ipynb.txt
+++ b/_sources/generated/demos/Main_Demo.ipynb.txt
@@ -429,6 +429,26 @@
                 "cv.attention.attention_patterns(tokens=gpt2_str_tokens, attention=attention_pattern)"
             ]
         },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "In this case, we only wanted the layer 0 attention patterns, but we are storing the internal activations from all locations in the model. It's convenient to have access to all activations, but this can be prohibitively expensive for memory use with larger models, batch sizes, or sequence lengths. In addition, we don't need to do the full forward pass through the model to collect layer 0 attention patterns. The following cell will collect only the layer 0 attention patterns and stop the forward pass at layer 1, requiring far less memory and compute."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "attn_hook_name = \"blocks.0.attn.hook_pattern\"\n",
+                "attn_layer = 0\n",
+                "_, gpt2_attn_cache = model.run_with_cache(gpt2_tokens, remove_batch_dim=True, stop_at_layer=attn_layer + 1, names_filter=[attn_hook_name])\n",
+                "gpt2_attn = gpt2_attn_cache[attn_hook_name]\n",
+                "assert torch.equal(gpt2_attn, attention_pattern)"
+            ]
+        },
         {
             "attachments": {},
             "cell_type": "markdown",
diff --git a/_sources/generated/model_properties_table.md.txt b/_sources/generated/model_properties_table.md.txt
index 30e018468..b653ccf5a 100644
--- a/_sources/generated/model_properties_table.md.txt
+++ b/_sources/generated/model_properties_table.md.txt
@@ -170,6 +170,21 @@
 | Qwen/Qwen2-1.5B-Instruct             | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
 | Qwen/Qwen2-7B                        | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
 | Qwen/Qwen2-7B-Instruct               | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-0.5B                    | 391M       |         24 |       896 |        14 | silu              |    2048 |    151936 |       64 |    4864 | 2                   |
+| Qwen/Qwen2.5-0.5B-Instruct           | 391M       |         24 |       896 |        14 | silu              |    2048 |    151936 |       64 |    4864 | 2                   |
+| Qwen/Qwen2.5-1.5B                    | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
+| Qwen/Qwen2.5-1.5B-Instruct           | 1.4B       |         28 |      1536 |        12 | silu              |    2048 |    151936 |      128 |    8960 | 2                   |
+| Qwen/Qwen2.5-3B                      | 3.0B       |         36 |      2048 |        16 | silu              |    2048 |    151936 |      128 |   11008 | 2                   |
+| Qwen/Qwen2.5-3B-Instruct             | 3.0B       |         36 |      2048 |        16 | silu              |    2048 |    151936 |      128 |   11008 | 2                   |
+| Qwen/Qwen2.5-7B                      | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-7B-Instruct             | 7.1B       |         28 |      3584 |        28 | silu              |    2048 |    152064 |      128 |   18944 | 4                   |
+| Qwen/Qwen2.5-14B                     | 15B        |         48 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   13824 | 8                   |
+| Qwen/Qwen2.5-14B-Instruct            | 15B        |         48 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   13824 | 8                   |
+| Qwen/Qwen2.5-32B                     | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
+| Qwen/Qwen2.5-32B-Instruct            | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
+| Qwen/Qwen2.5-72B                     | 80B        |         80 |      8192 |        64 | silu              |    2048 |    152064 |      128 |   29568 | 8                   |
+| Qwen/Qwen2.5-72B-Instruct            | 80B        |         80 |      8192 |        64 | silu              |    2048 |    152064 |      128 |   29568 | 8                   |
+| Qwen/QwQ-32B-Preview                 | 34B        |         64 |      5120 |        40 | silu              |    2048 |    152064 |      128 |   27648 | 8                   |
 | phi-1                                | 1.2B       |         24 |      2048 |        32 | gelu              |    2048 |     51200 |       64 |    8192 |                     |
 | phi-1_5                              | 1.2B       |         24 |      2048 |        32 | gelu              |    2048 |     51200 |       64 |    8192 |                     |
 | phi-2                                | 2.5B       |         32 |      2560 |        32 | gelu              |    2048 |     51200 |       80 |   10240 |                     |
diff --git a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
index cb9510a2f..bfe5dd961 100644
--- a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
index 51f82c9c8..c1488aa23 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
index 6198e655c..9638d4f3e 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_mlp_py.html b/_static/coverage/d_37285d613390727b_mlp_py.html
index 780c1cefa..cb99580ff 100644
--- a/_static/coverage/d_37285d613390727b_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_moe_py.html b/_static/coverage/d_37285d613390727b_moe_py.html
index 7576f0c39..2f9b7588f 100644
--- a/_static/coverage/d_37285d613390727b_moe_py.html
+++ b/_static/coverage/d_37285d613390727b_moe_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -204,7 +204,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
index 9ba5137e0..f337a8c69 100644
--- a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -128,7 +128,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
index 3624d0563..39e888b6e 100644
--- a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -112,7 +112,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_712808f24eb400fe___init___py.html b/_static/coverage/d_712808f24eb400fe___init___py.html
index 233e000b3..b3b5558a1 100644
--- a/_static/coverage/d_712808f24eb400fe___init___py.html
+++ b/_static/coverage/d_712808f24eb400fe___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
index 2af8de250..444524681 100644
--- a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
+++ b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">288 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">281<span class="text"> run</span></button>
+            <span class="text">289 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">282<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">7<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">10<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -104,1085 +104,1089 @@ <h2>
     <p class="run"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t"><span class="key">import</span> <span class="nam">einops</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t"><span class="key">import</span> <span class="nam">numpy</span> <span class="key">as</span> <span class="nam">np</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"><span class="key">from</span> <span class="nam">fancy_einsum</span> <span class="key">import</span> <span class="nam">einsum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span><span class="op">,</span> <span class="nam">Int</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">as</span> <span class="nam">utils</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span><span class="op">,</span> <span class="nam">Int</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">as</span> <span class="nam">utils</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t"><span class="key">class</span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">    <span class="str">"""Activation Cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"><span class="str">    A wrapper that stores all important activations from a forward pass of the model, and provides a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="str">    variety of helper functions to investigate them.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str">    The :class:`ActivationCache` is at the core of Transformer Lens. It is a wrapper that stores all</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str">    important activations from a forward pass of the model, and provides a variety of helper</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str">    functions to investigate them. The common way to access it is to run the model with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str">    :meth:`transformer_lens.HookedTransformer.run_with_cache`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="str">    Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t"><span class="str">    When investigating a particular behaviour of a modal, a very common first step is to try and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"><span class="str">    understand which components of the model are most responsible for that behaviour. For example,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"><span class="str">    if you're investigating the prompt "Why did the chicken cross the" -> " road", you might want to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t"><span class="str">    understand if there is a specific sublayer (mlp or multi-head attention) that is responsible for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"><span class="str">    the model predicting "road". This kind of analysis commonly falls under the category of "logit</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t"><span class="str">    attribution" or "direct logit attribution" (DLA).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"><span class="str">    >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="str">    >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"><span class="str">    Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="str">    >>> _logits, cache = model.run_with_cache("Why did the chicken cross the")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"><span class="str">    >>> residual_stream, labels = cache.decompose_resid(return_labels=True, mode="attn")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str">    >>> print(labels[0:3])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="str">    ['embed', 'pos_embed', '0_attn_out']</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="str">    >>> answer = " road" # Note the proceeding space to match the model's tokenization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="str">    >>> logit_attrs = cache.logit_attrs(residual_stream, answer)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"><span class="str">    >>> print(logit_attrs.shape) # Attention layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"><span class="str">    torch.Size([10, 1, 7])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t"><span class="str">    >>> most_important_component_idx = torch.argmax(logit_attrs)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"><span class="str">    >>> print(labels[most_important_component_idx])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"><span class="str">    3_attn_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="str">    You can also dig in with more granularity, using :meth:`get_full_resid_decomposition` to get the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str">    residual stream by individual component (mlp neurons and individual attention heads). This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="str">    creates a larger residual stack, but the approach of using :meth"`logit_attrs` remains the same.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"><span class="str">    Equally you might want to find out if the model struggles to construct such excellent jokes</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"><span class="str">    until the very last layers, or if it is trivial and the first few layers are enough. This kind</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"><span class="str">    of analysis is called "logit lens", and you can find out more about how to do that with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t"><span class="str">    :meth:`ActivationCache.accumulated_resid`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t"><span class="str">    Warning:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"><span class="str">    :class:`ActivationCache` is designed to be used with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="str">    :class:`transformer_lens.HookedTransformer`, and will not work with other models. It's also</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"><span class="str">    designed to be used with all activations of :class:`transformer_lens.HookedTransformer` being</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t"><span class="str">    cached, and some internal methods will break without that.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t"><span class="str">    The biggest footgun and source of bugs in this code will be keeping track of indexes,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t"><span class="str">    dimensions, and the numbers of each. There are several kinds of activations:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t"><span class="str">    * Internal attn head vectors: q, k, v, z. Shape [batch, pos, head_index, d_head].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t"><span class="str">    * Internal attn pattern style results: pattern (post softmax), attn_scores (pre-softmax). Shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t"><span class="str">      [batch, head_index, query_pos, key_pos].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t"><span class="str">    * Attn head results: result. Shape [batch, pos, head_index, d_model].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t"><span class="str">    * Internal MLP vectors: pre, post, mid (only used for solu_ln - the part between activation +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t"><span class="str">      layernorm). Shape [batch, pos, d_mlp].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t"><span class="str">    * Residual stream vectors: resid_pre, resid_mid, resid_post, attn_out, mlp_out, embed,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t"><span class="str">      pos_embed, normalized (output of each LN or LNPre). Shape [batch, pos, d_model].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t"><span class="str">    * LayerNorm Scale: scale. Shape [batch, pos, 1].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"><span class="str">    Sometimes the batch dimension will be missing because we applied `remove_batch_dim` (used when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t"><span class="str">    batch_size=1), and as such all library functions *should* be robust to that.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"><span class="str">    Type annotations are in the following form:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str">    * layers_covered is the number of layers queried in functions that stack the residual stream.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="str">    * batch_and_pos_dims is the set of dimensions from batch and pos - by default this is ["batch",</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str">      "pos"], but is only ["pos"] if we've removed the batch dimension and is [()] if we've removed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str">      batch dimension and are applying a pos slice which indexes a specific position.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t"><span class="str">        cache_dict:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t"><span class="str">            A dictionary of cached activations from a model run.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t"><span class="str">        model:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t"><span class="str">            The model that the activations are from.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t"><span class="str">        has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t"><span class="str">            Whether the activations have a batch dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">cache_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">model</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span> <span class="op">=</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">model</span> <span class="op">=</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="op">=</span> <span class="nam">has_batch_dim</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span> <span class="op">=</span> <span class="str">"hook_embed"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span> <span class="op">=</span> <span class="str">"hook_pos_embed"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t">    <span class="key">def</span> <span class="nam">remove_batch_dim</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t">        <span class="str">"""Remove the Batch Dimension (if a single batch item).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"><span class="str">            The ActivationCache with the batch dimension removed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="num">0</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot remove batch dimension from cache with batch size > 1, \</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t"><span class="str">                    for key {key} with shape {self.cache_dict[key].shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Tried removing batch dimension after already having removed it."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">    <span class="key">def</span> <span class="nam">__repr__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">        <span class="str">"""Representation of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t"><span class="str">        Special method that returns a string representation of an object. It's normally used to give</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t"><span class="str">        a string that can be used to recreate the object, but here we just return a string that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t"><span class="str">        describes the object.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">        <span class="key">return</span> <span class="str">f"ActivationCache with keys {list(self.cache_dict.keys())}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">    <span class="key">def</span> <span class="nam">__getitem__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">key</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">        <span class="str">"""Retrieve Cached Activations by Key or Shorthand.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t"><span class="str">        Enables direct access to cached activations via dictionary-style indexing using keys or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t"><span class="str">        shorthand naming conventions. It also supports tuples for advanced indexing, with the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t"><span class="str">        dimension order as (get_act_name, layer_index, layer_type).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t"><span class="str">            key:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t"><span class="str">                The key or shorthand name for the activation to retrieve.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t"><span class="str">            The cached activation tensor corresponding to the given key.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">        <span class="key">if</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">        <span class="key">elif</span> <span class="nam">type</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">utils</span><span class="op">.</span><span class="nam">get_act_name</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">            <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span> <span class="op">></span> <span class="num">1</span> <span class="key">and</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">                <span class="key">if</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span> <span class="op">&lt;</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">                    <span class="com"># Supports negative indexing on the layer dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">                    <span class="nam">key</span> <span class="op">=</span> <span class="op">(</span><span class="nam">key</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">+</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span> <span class="op">*</span><span class="nam">key</span><span class="op">[</span><span class="num">2</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">utils</span><span class="op">.</span><span class="nam">get_act_name</span><span class="op">(</span><span class="op">*</span><span class="nam">key</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">    <span class="key">def</span> <span class="nam">__len__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">int</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">        <span class="str">"""Length of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t"><span class="str">        Special method that returns the length of an object (in this case the number of different</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t"><span class="str">        activations in the cache).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">        <span class="key">return</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">    <span class="key">def</span> <span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">,</span> <span class="nam">move_model</span><span class="op">=</span><span class="key">False</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">        <span class="str">"""Move the Cache to a Device.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t"><span class="str">        Mostly useful for moving the cache to the CPU after model computation finishes to save GPU</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t"><span class="str">        memory. Note however that operations will be much slower on the CPU. Note also that some</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t"><span class="str">        methods will break unless the model is also moved to the same device, eg</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t"><span class="str">        `compute_head_results`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t"><span class="str">            device:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t"><span class="str">                The device to move the cache to (e.g. `torch.device.cpu`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t"><span class="str">            move_model:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t"><span class="str">                Whether to also move the model to the same device. @deprecated</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">        <span class="com"># Move model is deprecated as we plan on de-coupling the classes</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">            <span class="nam">warnings</span><span class="op">.</span><span class="nam">warn</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">                <span class="str">"The 'move_model' parameter is deprecated."</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">                <span class="nam">DeprecationWarning</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">value</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span> <span class="key">for</span> <span class="nam">key</span><span class="op">,</span> <span class="nam">value</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_model</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">    <span class="key">def</span> <span class="nam">toggle_autodiff</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">        <span class="str">"""Toggle Autodiff Globally.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t"><span class="str">        Applies `torch.set_grad_enabled(mode)` to the global state (not just TransformerLens).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t"><span class="str">        Warning:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t"><span class="str">        This is pretty dangerous, since autodiff is global state - this turns off torch's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t"><span class="str">        ability to take gradients completely and it's easy to get a bunch of errors if you don't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t"><span class="str">        realise what you're doing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t"><span class="str">        But autodiff consumes a LOT of GPU memory (since every intermediate activation is cached</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t"><span class="str">        until all downstream activations are deleted - this means that computing the loss and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t"><span class="str">        storing it in a list will keep every activation sticking around!). So often when you're</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t"><span class="str">        analysing a model's activations, and don't need to do any training, autodiff is more trouble</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t"><span class="str">        than its worth.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t"><span class="str">        If you don't want to mess with global state, using torch.inference_mode as a context manager</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t"><span class="str">        or decorator achieves similar effects:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t"><span class="str">        >>> with torch.inference_mode():</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t"><span class="str">        ...     y = torch.Tensor([1., 2, 3])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t"><span class="str">        >>> y.requires_grad</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t"><span class="str">        False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Changed the global state, set autodiff to %s"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">        <span class="nam">torch</span><span class="op">.</span><span class="nam">set_grad_enabled</span><span class="op">(</span><span class="nam">mode</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">    <span class="key">def</span> <span class="nam">keys</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">        <span class="str">"""Keys of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t"><span class="str">            >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t"><span class="str">            >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t"><span class="str">            Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t"><span class="str">            >>> _logits, cache = model.run_with_cache("Some prompt")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t"><span class="str">            >>> list(cache.keys())[0:3]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t"><span class="str">            ['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre']</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t"><span class="str">            List of all keys.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">    <span class="key">def</span> <span class="nam">values</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">        <span class="str">"""Values of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t"><span class="str">            List of all values.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="key">def</span> <span class="nam">items</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">        <span class="str">"""Items of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t"><span class="str">            List of all items ((key, value) tuples).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">    <span class="key">def</span> <span class="nam">__iter__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Iterator</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">        <span class="str">"""ActivationCache Iterator.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t"><span class="str">        Special method that returns an iterator over the ActivationCache. Allows looping over the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t"><span class="str">        cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t"><span class="str">            >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t"><span class="str">            >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t"><span class="str">            Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t"><span class="str">            >>> _logits, cache = model.run_with_cache("Some prompt")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t"><span class="str">            >>> cache_interesting_names = []</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t"><span class="str">            >>> for key in cache:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t"><span class="str">            ...     if not key.startswith("blocks.") or key.startswith("blocks.0"):</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t"><span class="str">            ...         cache_interesting_names.append(key)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t"><span class="str">            >>> print(cache_interesting_names[0:3])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t"><span class="str">            ['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre']</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t"><span class="str">            Iterator over the cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">__iter__</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_slice_to_batch_dim</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">        <span class="str">"""Apply a Slice to the Batch Dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t"><span class="str">                The slice to apply to the batch dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t"><span class="str">            The ActivationCache with the batch dimension sliced.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">batch_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="key">or</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">mode</span> <span class="op">==</span> <span class="str">"empty"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Cannot index into a cache without a batch dim"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">        <span class="nam">still_has_batch_dim</span> <span class="op">=</span> <span class="op">(</span><span class="nam">batch_slice</span><span class="op">.</span><span class="nam">mode</span> <span class="op">!=</span> <span class="str">"int"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">        <span class="nam">new_cache_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">            <span class="nam">name</span><span class="op">:</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span> <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">        <span class="key">return</span> <span class="nam">ActivationCache</span><span class="op">(</span><span class="nam">new_cache_dict</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">=</span><span class="nam">still_has_batch_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">        <span class="nam">incl_mid</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="str">"""Accumulated Residual Stream.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t"><span class="str">        Returns the accumulated residual stream at each layer/sub-layer. This is useful for `Logit</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t"><span class="str">        Lens &lt;https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens>`</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t"><span class="str">        style analysis, where it can be thought of as what the model "believes" at each point in the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t"><span class="str">        residual stream.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t"><span class="str">        To project this into the vocabulary space, remember that there is a final layer norm in most</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t"><span class="str">        decoder-only transformers. Therefore, you need to first apply the final layer norm (which</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t"><span class="str">        can be done with `apply_ln`), and then multiply by the unembedding matrix (:math:`W_U`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t"><span class="str">        If you instead want to look at contributions to the residual stream from each component</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t"><span class="str">        (e.g. for direct logit attribution), see :meth:`decompose_resid` instead, or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t"><span class="str">        :meth:`get_full_resid_decomposition` if you want contributions broken down further into each</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t"><span class="str">        MLP neuron.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t"><span class="str">        Logit Lens analysis can be done as follows:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t"><span class="str">        >>> from einops import einsum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t"><span class="str">        >>> import torch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t"><span class="str">        >>> import pandas as pd</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M", device="cpu")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t"><span class="str">        >>> prompt = "Why did the chicken cross the"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t"><span class="str">        >>> answer = " road"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t"><span class="str">        >>> logits, cache = model.run_with_cache("Why did the chicken cross the")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t"><span class="str">        >>> answer_token = model.to_single_token(answer)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t"><span class="str">        >>> print(answer_token)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t"><span class="str">        2975</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t"><span class="str">        >>> accum_resid, labels = cache.accumulated_resid(return_labels=True, apply_ln=True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t"><span class="str">        >>> last_token_accum = accum_resid[:, 0, -1, :]  # layer, batch, pos, d_model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t"><span class="str">        >>> print(last_token_accum.shape)  # layer, d_model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t"><span class="str">        torch.Size([9, 64])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t"><span class="str">        >>> W_U = model.W_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t"><span class="str">        >>> print(W_U.shape)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t"><span class="str">        torch.Size([64, 50257])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t"><span class="str">        >>> layers_unembedded = einsum(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t"><span class="str">        ...         last_token_accum,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t"><span class="str">        ...         W_U,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t"><span class="str">        ...         "layer d_model, d_model d_vocab -> layer d_vocab"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t"><span class="str">        ...     )</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t"><span class="str">        >>> print(layers_unembedded.shape)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t"><span class="str">        torch.Size([9, 50257])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t"><span class="str">        >>> # Get the rank of the correct answer by layer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t"><span class="str">        >>> sorted_indices = torch.argsort(layers_unembedded, dim=1, descending=True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t"><span class="str">        >>> rank_answer = (sorted_indices == 2975).nonzero(as_tuple=True)[1]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t"><span class="str">        >>> print(pd.Series(rank_answer, index=labels))</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t"><span class="str">        0_pre         4442</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t"><span class="str">        1_pre          382</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t"><span class="str">        2_pre          982</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t"><span class="str">        3_pre         1160</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t"><span class="str">        4_pre          408</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t"><span class="str">        5_pre          145</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t"><span class="str">        6_pre           78</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t"><span class="str">        7_pre          387</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t"><span class="str">        final_post       6</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t"><span class="str">        dtype: int64</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t"><span class="str">                The layer to take components up to - by default includes resid_pre for that layer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t"><span class="str">                and excludes resid_mid and resid_post for that layer. If set as `n_layers`, `-1` or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t"><span class="str">                `None` it will return all residual streams, including the final one (i.e.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t"><span class="str">                immediately pre logits). The indices are taken such that this gives the accumulated</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t"><span class="str">                streams up to the input to layer l.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t"><span class="str">            incl_mid:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t"><span class="str">                Whether to return `resid_mid` for all previous layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t"><span class="str">                Whether to include resid_mid for the current layer. This essentially gives the MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t"><span class="str">                input rather than the attention input.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t"><span class="str">                Whether to return a list of labels for the residual stream components. Useful for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t"><span class="str">                labelling graphs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t"><span class="str">            A tensor of the accumulated residual streams. If `return_labels` is True, also returns a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t"><span class="str">            list of labels for the components (as a tuple in the form `(components, labels)`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span> <span class="op">+</span> <span class="num">1</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">            <span class="key">if</span> <span class="nam">l</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"final_post"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">                <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">            <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_pre"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_pre"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span><span class="nam">incl_mid</span> <span class="key">and</span> <span class="nam">l</span> <span class="op">&lt;</span> <span class="nam">layer</span><span class="op">)</span> <span class="key">or</span> <span class="op">(</span><span class="nam">mlp_input</span> <span class="key">and</span> <span class="nam">l</span> <span class="op">==</span> <span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_mid"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_mid"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">c</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span> <span class="key">for</span> <span class="nam">c</span> <span class="key">in</span> <span class="nam">components_list</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">    <span class="key">def</span> <span class="nam">logit_attrs</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">        <span class="nam">residual_stack</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">            <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch position"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="nam">incorrect_tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">            <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">                <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">                <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch position"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">        <span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims_out"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">        <span class="str">"""Logit Attributions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t"><span class="str">        Takes a residual stack (typically the residual stream decomposed by components), and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t"><span class="str">        calculates how much each item in the stack "contributes" to specific tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t"><span class="str">        It does this by:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t"><span class="str">            1. Getting the residual directions of the tokens (i.e. reversing the unembed)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t"><span class="str">            2. Taking the dot product of each item in the residual stack, with the token residual</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t"><span class="str">               directions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t"><span class="str">        Note that if incorrect tokens are provided, it instead takes the difference between the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t"><span class="str">        correct and incorrect tokens (to calculate the residual directions). This is useful as</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t"><span class="str">        sometimes we want to know e.g. which components are most responsible for selecting the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t"><span class="str">        correct token rather than an incorrect one. For example in the `Interpretability in the Wild</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t"><span class="str">        paper &lt;https://arxiv.org/abs/2211.00593>` prompts such as "John and Mary went to the shops,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t"><span class="str">        John gave a bag to" were investigated, and it was therefore useful to calculate attribution</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t"><span class="str">        for the :math:`\\text{Mary} - \\text{John}` residual direction.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t"><span class="str">        Warning:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t"><span class="str">        Choosing the correct `tokens` and `incorrect_tokens` is both important and difficult. When</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t"><span class="str">        investigating specific components it's also useful to look at it's impact on all tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t"><span class="str">        (i.e. :math:`\\text{final_ln}(\\text{residual_stack_item}) W_U`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t"><span class="str">            residual_stack:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t"><span class="str">                Stack of components of residual stream to get logit attributions for.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t"><span class="str">            tokens:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t"><span class="str">                Tokens to compute logit attributions on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t"><span class="str">            incorrect_tokens:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t"><span class="str">                If provided, compute attributions on logit difference between tokens and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t"><span class="str">                incorrect_tokens. Must have the same shape as tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t"><span class="str">                The slice to apply layer norm scaling on. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t"><span class="str">                The slice to take on the batch dimension during layer norm scaling. Defaults to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t"><span class="str">                None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t"><span class="str">            has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t"><span class="str">                Whether residual_stack has a batch dimension. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t"><span class="str">            A tensor of the logit attributions or logit difference attributions if incorrect_tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t"><span class="str">            was provided.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">        <span class="nam">logit_directions</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">tokens_to_residual_directions</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">        <span class="key">if</span> <span class="nam">incorrect_tokens</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">                <span class="nam">incorrect_tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">                <span class="nam">incorrect_tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">            <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span> <span class="op">!=</span> <span class="nam">incorrect_tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">                    <span class="str">f"tokens and incorrect_tokens must have the same shape! \</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t"><span class="str">                        (tokens.shape={tokens.shape}, \</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t"><span class="str">                        incorrect_tokens.shape={incorrect_tokens.shape})"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">            <span class="com"># If incorrect_tokens was provided, take the logit difference</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">            <span class="nam">logit_directions</span> <span class="op">=</span> <span class="nam">logit_directions</span> <span class="op">-</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">tokens_to_residual_directions</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">                <span class="nam">incorrect_tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="nam">scaled_residual_stack</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">            <span class="nam">residual_stack</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">            <span class="nam">layer</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">            <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">            <span class="nam">batch_slice</span><span class="op">=</span><span class="nam">batch_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">            <span class="nam">has_batch_dim</span><span class="op">=</span><span class="nam">has_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="nam">logit_attrs</span> <span class="op">=</span> <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="str">"... d_model, ... d_model -> ..."</span><span class="op">,</span> <span class="nam">scaled_residual_stack</span><span class="op">,</span> <span class="nam">logit_directions</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">        <span class="key">return</span> <span class="nam">logit_attrs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">    <span class="key">def</span> <span class="nam">decompose_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">        <span class="nam">mode</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"all"</span><span class="op">,</span> <span class="str">"mlp"</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"all"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">        <span class="nam">incl_embeds</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">        <span class="str">"""Decompose the Residual Stream.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t"><span class="str">        Decomposes the residual stream input to layer L into a stack of the output of previous</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t"><span class="str">        layers. The sum of these is the input to layer L (plus embedding and pos embedding). This is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t"><span class="str">        useful for attributing model behaviour to different components of the residual stream</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">                The layer to take components up to - by default includes</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t"><span class="str">                resid_pre for that layer and excludes resid_mid and resid_post for that layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t"><span class="str">                layer==n_layers means to return all layer outputs incl in the final layer, layer==0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t"><span class="str">                means just embed and pos_embed. The indices are taken such that this gives the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t"><span class="str">                accumulated streams up to the input to layer l</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t"><span class="str">                Whether to include attn_out for the current</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">                layer - essentially decomposing the residual stream that's input to the MLP input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t"><span class="str">                rather than the Attn input.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t"><span class="str">            mode:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t"><span class="str">                Values are "all", "mlp" or "attn". "all" returns all</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t"><span class="str">                components, "mlp" returns only the MLP components, and "attn" returns only the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t"><span class="str">                attention components. Defaults to "all".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t"><span class="str">                Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t"><span class="str">            incl_embeds:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t"><span class="str">                Whether to include embed &amp; pos_embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t"><span class="str">                Whether to return a list of labels for the residual stream components.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t"><span class="str">                Useful for labelling graphs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t"><span class="str">            A tensor of the accumulated residual streams. If `return_labels` is True, also returns</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t"><span class="str">            a list of labels for the components (as a tuple in the form `(components, labels)`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="nam">incl_attn</span> <span class="op">=</span> <span class="nam">mode</span> <span class="op">!=</span> <span class="str">"mlp"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">        <span class="nam">incl_mlp</span> <span class="op">=</span> <span class="nam">mode</span> <span class="op">!=</span> <span class="str">"attn"</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">        <span class="key">if</span> <span class="nam">incl_embeds</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">628&#x202F;&#x219B;&#x202F;631</span><span class="annotate long">line 628 didn't jump to line 631, because the condition on line 628 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">                <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">631&#x202F;&#x219B;&#x202F;635</span><span class="annotate long">line 631 didn't jump to line 635, because the condition on line 631 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_pos_embed"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"pos_embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_attn</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"attn_out"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_attn_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"mlp_out"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_mlp_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span> <span class="key">and</span> <span class="nam">incl_attn</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">            <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"attn_out"</span><span class="op">,</span> <span class="nam">layer</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{layer}_attn_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">c</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span> <span class="key">for</span> <span class="nam">c</span> <span class="key">in</span> <span class="nam">components_list</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">    <span class="key">def</span> <span class="nam">compute_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">        <span class="str">"""Compute Head Results.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t"><span class="str">        Computes and caches the results for each attention head, ie the amount contributed to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t"><span class="str">        residual stream from that head. attn_out for a layer is the sum of head results plus b_O.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t"><span class="str">        Intended use is to enable use_attn_results when running and caching the model, but this can</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t"><span class="str">        be useful if you forget.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">        <span class="key">if</span> <span class="str">"blocks.0.attn.hook_result"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Tried to compute head results when they were already cached"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">            <span class="key">return</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">            <span class="com"># Note that we haven't enabled set item on this object so we need to edit the underlying</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">            <span class="com"># cache_dict directly.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.hook_result"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">                <span class="str">"... head_index d_head, head_index d_head d_model -> ... head_index d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">                <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"z"</span><span class="op">,</span> <span class="nam">l</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">)</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">l</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">        <span class="nam">incl_remainder</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">        <span class="str">"""Stack Head Results.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t"><span class="str">        Returns a stack of all head results (ie residual stream contribution) up to layer L. A good</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t"><span class="str">        way to decompose the outputs of attention layers into attribution by specific heads. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t"><span class="str">        that the num_components axis has length layer x n_heads ((layer head_index) in einops</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t"><span class="str">        notation).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t"><span class="str">                Layer index - heads at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t"><span class="str">                in [1, n_layers-1], or any of (n_layers, -1, None), which all mean the final layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t"><span class="str">                Whether to also return a list of labels of the form "L0H0" for the heads.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t">        <span class="key">if</span> <span class="str">"blocks.0.attn.hook_result"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">            <span class="nam">print</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">                <span class="str">"Tried to stack head results when they weren't cached. Computing head results now"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">compute_head_results</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">        <span class="nam">components</span><span class="op">:</span> <span class="nam">Any</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">            <span class="com"># Note that this has shape batch x pos x head_index x d_model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"result"</span><span class="op">,</span> <span class="nam">l</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">3</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t">        <span class="key">if</span> <span class="nam">components</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t">                <span class="str">"... concat_head_index d_model -> concat_head_index ... d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t">                <span class="nam">remainder</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">                    <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">                <span class="op">)</span> <span class="op">-</span> <span class="nam">components</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">                <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">components</span><span class="op">,</span> <span class="nam">remainder</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">        <span class="key">elif</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">            <span class="com"># There are no components, so the remainder is the entire thing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">                <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">            <span class="com"># If this is called with layer 0, we return an empty tensor of the right shape to be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">            <span class="com"># stacked correctly. This uses the shape of hook_embed, which is pretty janky since it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">            <span class="com"># assumes embed is in the cache. But it's hard to explicitly code the shape, since it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">            <span class="com"># depends on the pos slice, whether we have a batch dim, etc. And it's pretty messy!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">                <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">                <span class="op">*</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">                <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_activation</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">        <span class="nam">activation_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">        <span class="nam">sublayer_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered ..."</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">        <span class="str">"""Stack Activations.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t"><span class="str">        Flexible way to stack activations with a given name.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t"><span class="str">            activation_name:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t"><span class="str">                The name of the activation to be stacked</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t"><span class="str">                'Layer index - heads' at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t"><span class="str">                in [1, n_layers-1], or any of (n_layers, -1, None), which all mean the final layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t"><span class="str">            sublayer_type:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t"><span class="str">                The sub layer type of the activation, passed to utils.get_act_name. Can normally be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t"><span class="str">                inferred.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="nam">activation_name</span><span class="op">,</span> <span class="nam">l</span><span class="op">,</span> <span class="nam">sublayer_type</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="key">class</span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">    <span class="str">"""Activation Cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t"><span class="str">    A wrapper that stores all important activations from a forward pass of the model, and provides a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"><span class="str">    variety of helper functions to investigate them.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t"><span class="str">    The :class:`ActivationCache` is at the core of Transformer Lens. It is a wrapper that stores all</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str">    important activations from a forward pass of the model, and provides a variety of helper</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str">    functions to investigate them. The common way to access it is to run the model with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str">    :meth:`transformer_lens.HookedTransformer.run_with_cache`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="str">    Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t"><span class="str">    When investigating a particular behaviour of a modal, a very common first step is to try and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t"><span class="str">    understand which components of the model are most responsible for that behaviour. For example,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"><span class="str">    if you're investigating the prompt "Why did the chicken cross the" -> " road", you might want to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"><span class="str">    understand if there is a specific sublayer (mlp or multi-head attention) that is responsible for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t"><span class="str">    the model predicting "road". This kind of analysis commonly falls under the category of "logit</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"><span class="str">    attribution" or "direct logit attribution" (DLA).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"><span class="str">    >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"><span class="str">    >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="str">    Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="str">    >>> _logits, cache = model.run_with_cache("Why did the chicken cross the")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="str">    >>> residual_stream, labels = cache.decompose_resid(return_labels=True, mode="attn")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"><span class="str">    >>> print(labels[0:3])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str">    ['embed', 'pos_embed', '0_attn_out']</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="str">    >>> answer = " road" # Note the proceeding space to match the model's tokenization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="str">    >>> logit_attrs = cache.logit_attrs(residual_stream, answer)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="str">    >>> print(logit_attrs.shape) # Attention layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"><span class="str">    torch.Size([10, 1, 7])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t"><span class="str">    >>> most_important_component_idx = torch.argmax(logit_attrs)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t"><span class="str">    >>> print(labels[most_important_component_idx])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"><span class="str">    3_attn_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="str">    You can also dig in with more granularity, using :meth:`get_full_resid_decomposition` to get the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="str">    residual stream by individual component (mlp neurons and individual attention heads). This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str">    creates a larger residual stack, but the approach of using :meth"`logit_attrs` remains the same.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"><span class="str">    Equally you might want to find out if the model struggles to construct such excellent jokes</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"><span class="str">    until the very last layers, or if it is trivial and the first few layers are enough. This kind</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"><span class="str">    of analysis is called "logit lens", and you can find out more about how to do that with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"><span class="str">    :meth:`ActivationCache.accumulated_resid`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t"><span class="str">    Warning:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t"><span class="str">    :class:`ActivationCache` is designed to be used with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"><span class="str">    :class:`transformer_lens.HookedTransformer`, and will not work with other models. It's also</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="str">    designed to be used with all activations of :class:`transformer_lens.HookedTransformer` being</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"><span class="str">    cached, and some internal methods will break without that.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t"><span class="str">    The biggest footgun and source of bugs in this code will be keeping track of indexes,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t"><span class="str">    dimensions, and the numbers of each. There are several kinds of activations:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t"><span class="str">    * Internal attn head vectors: q, k, v, z. Shape [batch, pos, head_index, d_head].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t"><span class="str">    * Internal attn pattern style results: pattern (post softmax), attn_scores (pre-softmax). Shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t"><span class="str">      [batch, head_index, query_pos, key_pos].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t"><span class="str">    * Attn head results: result. Shape [batch, pos, head_index, d_model].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t"><span class="str">    * Internal MLP vectors: pre, post, mid (only used for solu_ln - the part between activation +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t"><span class="str">      layernorm). Shape [batch, pos, d_mlp].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t"><span class="str">    * Residual stream vectors: resid_pre, resid_mid, resid_post, attn_out, mlp_out, embed,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t"><span class="str">      pos_embed, normalized (output of each LN or LNPre). Shape [batch, pos, d_model].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t"><span class="str">    * LayerNorm Scale: scale. Shape [batch, pos, 1].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t"><span class="str">    Sometimes the batch dimension will be missing because we applied `remove_batch_dim` (used when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"><span class="str">    batch_size=1), and as such all library functions *should* be robust to that.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t"><span class="str">    Type annotations are in the following form:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t"><span class="str">    * layers_covered is the number of layers queried in functions that stack the residual stream.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str">    * batch_and_pos_dims is the set of dimensions from batch and pos - by default this is ["batch",</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="str">      "pos"], but is only ["pos"] if we've removed the batch dimension and is [()] if we've removed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str">      batch dimension and are applying a pos slice which indexes a specific position.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t"><span class="str">        cache_dict:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t"><span class="str">            A dictionary of cached activations from a model run.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t"><span class="str">        model:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t"><span class="str">            The model that the activations are from.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t"><span class="str">        has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t"><span class="str">            Whether the activations have a batch dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">cache_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">model</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span> <span class="op">=</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">model</span> <span class="op">=</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="op">=</span> <span class="nam">has_batch_dim</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span> <span class="op">=</span> <span class="str">"hook_embed"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span> <span class="op">=</span> <span class="str">"hook_pos_embed"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t">    <span class="key">def</span> <span class="nam">remove_batch_dim</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t">        <span class="str">"""Remove the Batch Dimension (if a single batch item).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"><span class="str">            The ActivationCache with the batch dimension removed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="num">0</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot remove batch dimension from cache with batch size > 1, \</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t"><span class="str">                    for key {key} with shape {self.cache_dict[key].shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Tried removing batch dimension after already having removed it."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">    <span class="key">def</span> <span class="nam">__repr__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">        <span class="str">"""Representation of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t"><span class="str">        Special method that returns a string representation of an object. It's normally used to give</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t"><span class="str">        a string that can be used to recreate the object, but here we just return a string that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t"><span class="str">        describes the object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t">        <span class="key">return</span> <span class="str">f"ActivationCache with keys {list(self.cache_dict.keys())}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">    <span class="key">def</span> <span class="nam">__getitem__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">key</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">        <span class="str">"""Retrieve Cached Activations by Key or Shorthand.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t"><span class="str">        Enables direct access to cached activations via dictionary-style indexing using keys or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t"><span class="str">        shorthand naming conventions. It also supports tuples for advanced indexing, with the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t"><span class="str">        dimension order as (get_act_name, layer_index, layer_type).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t"><span class="str">            key:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t"><span class="str">                The key or shorthand name for the activation to retrieve.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t"><span class="str">            The cached activation tensor corresponding to the given key.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t">        <span class="key">if</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">        <span class="key">elif</span> <span class="nam">type</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">utils</span><span class="op">.</span><span class="nam">get_act_name</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">            <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">key</span><span class="op">)</span> <span class="op">></span> <span class="num">1</span> <span class="key">and</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">                <span class="key">if</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span> <span class="op">&lt;</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">                    <span class="com"># Supports negative indexing on the layer dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">                    <span class="nam">key</span> <span class="op">=</span> <span class="op">(</span><span class="nam">key</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">+</span> <span class="nam">key</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span> <span class="op">*</span><span class="nam">key</span><span class="op">[</span><span class="num">2</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="nam">utils</span><span class="op">.</span><span class="nam">get_act_name</span><span class="op">(</span><span class="op">*</span><span class="nam">key</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">    <span class="key">def</span> <span class="nam">__len__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">int</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">        <span class="str">"""Length of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t"><span class="str">        Special method that returns the length of an object (in this case the number of different</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t"><span class="str">        activations in the cache).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t">        <span class="key">return</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">    <span class="key">def</span> <span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">,</span> <span class="nam">move_model</span><span class="op">=</span><span class="key">False</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">        <span class="str">"""Move the Cache to a Device.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t"><span class="str">        Mostly useful for moving the cache to the CPU after model computation finishes to save GPU</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t"><span class="str">        memory. Note however that operations will be much slower on the CPU. Note also that some</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t"><span class="str">        methods will break unless the model is also moved to the same device, eg</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t"><span class="str">        `compute_head_results`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t"><span class="str">            device:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t"><span class="str">                The device to move the cache to (e.g. `torch.device.cpu`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t"><span class="str">            move_model:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t"><span class="str">                Whether to also move the model to the same device. @deprecated</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">        <span class="com"># Move model is deprecated as we plan on de-coupling the classes</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">            <span class="nam">warnings</span><span class="op">.</span><span class="nam">warn</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">                <span class="str">"The 'move_model' parameter is deprecated."</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">                <span class="nam">DeprecationWarning</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">value</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span> <span class="key">for</span> <span class="nam">key</span><span class="op">,</span> <span class="nam">value</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_model</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">    <span class="key">def</span> <span class="nam">toggle_autodiff</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">        <span class="str">"""Toggle Autodiff Globally.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t"><span class="str">        Applies `torch.set_grad_enabled(mode)` to the global state (not just TransformerLens).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t"><span class="str">        Warning:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t"><span class="str">        This is pretty dangerous, since autodiff is global state - this turns off torch's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t"><span class="str">        ability to take gradients completely and it's easy to get a bunch of errors if you don't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t"><span class="str">        realise what you're doing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t"><span class="str">        But autodiff consumes a LOT of GPU memory (since every intermediate activation is cached</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t"><span class="str">        until all downstream activations are deleted - this means that computing the loss and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t"><span class="str">        storing it in a list will keep every activation sticking around!). So often when you're</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t"><span class="str">        analysing a model's activations, and don't need to do any training, autodiff is more trouble</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t"><span class="str">        than its worth.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t"><span class="str">        If you don't want to mess with global state, using torch.inference_mode as a context manager</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t"><span class="str">        or decorator achieves similar effects:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t"><span class="str">        >>> with torch.inference_mode():</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t"><span class="str">        ...     y = torch.Tensor([1., 2, 3])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t"><span class="str">        >>> y.requires_grad</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t"><span class="str">        False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Changed the global state, set autodiff to %s"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">        <span class="nam">torch</span><span class="op">.</span><span class="nam">set_grad_enabled</span><span class="op">(</span><span class="nam">mode</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">    <span class="key">def</span> <span class="nam">keys</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">        <span class="str">"""Keys of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t"><span class="str">            >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t"><span class="str">            >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t"><span class="str">            Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t"><span class="str">            >>> _logits, cache = model.run_with_cache("Some prompt")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t"><span class="str">            >>> list(cache.keys())[0:3]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t"><span class="str">            ['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre']</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t"><span class="str">            List of all keys.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">    <span class="key">def</span> <span class="nam">values</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">        <span class="str">"""Values of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t"><span class="str">            List of all values.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">    <span class="key">def</span> <span class="nam">items</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">        <span class="str">"""Items of the ActivationCache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t"><span class="str">            List of all items ((key, value) tuples).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">    <span class="key">def</span> <span class="nam">__iter__</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Iterator</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">        <span class="str">"""ActivationCache Iterator.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t"><span class="str">        Special method that returns an iterator over the ActivationCache. Allows looping over the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t"><span class="str">        cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t"><span class="str">            >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t"><span class="str">            >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t"><span class="str">            Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t"><span class="str">            >>> _logits, cache = model.run_with_cache("Some prompt")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t"><span class="str">            >>> cache_interesting_names = []</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t"><span class="str">            >>> for key in cache:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t"><span class="str">            ...     if not key.startswith("blocks.") or key.startswith("blocks.0"):</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t"><span class="str">            ...         cache_interesting_names.append(key)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t"><span class="str">            >>> print(cache_interesting_names[0:3])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t"><span class="str">            ['hook_embed', 'hook_pos_embed', 'blocks.0.hook_resid_pre']</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t"><span class="str">            Iterator over the cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">__iter__</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_slice_to_batch_dim</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">)</span> <span class="op">-></span> <span class="nam">ActivationCache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="str">"""Apply a Slice to the Batch Dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t"><span class="str">                The slice to apply to the batch dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t"><span class="str">            The ActivationCache with the batch dimension sliced.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">        <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">batch_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span> <span class="key">or</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">mode</span> <span class="op">==</span> <span class="str">"empty"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Cannot index into a cache without a batch dim"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">        <span class="nam">still_has_batch_dim</span> <span class="op">=</span> <span class="op">(</span><span class="nam">batch_slice</span><span class="op">.</span><span class="nam">mode</span> <span class="op">!=</span> <span class="str">"int"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">        <span class="nam">new_cache_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">            <span class="nam">name</span><span class="op">:</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span> <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">        <span class="key">return</span> <span class="nam">ActivationCache</span><span class="op">(</span><span class="nam">new_cache_dict</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">=</span><span class="nam">still_has_batch_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="nam">incl_mid</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="str">"""Accumulated Residual Stream.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t"><span class="str">        Returns the accumulated residual stream at each layer/sub-layer. This is useful for `Logit</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t"><span class="str">        Lens &lt;https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens>`</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t"><span class="str">        style analysis, where it can be thought of as what the model "believes" at each point in the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t"><span class="str">        residual stream.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t"><span class="str">        To project this into the vocabulary space, remember that there is a final layer norm in most</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t"><span class="str">        decoder-only transformers. Therefore, you need to first apply the final layer norm (which</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t"><span class="str">        can be done with `apply_ln`), and then multiply by the unembedding matrix (:math:`W_U`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t"><span class="str">        If you instead want to look at contributions to the residual stream from each component</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t"><span class="str">        (e.g. for direct logit attribution), see :meth:`decompose_resid` instead, or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t"><span class="str">        :meth:`get_full_resid_decomposition` if you want contributions broken down further into each</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t"><span class="str">        MLP neuron.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t"><span class="str">        Logit Lens analysis can be done as follows:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t"><span class="str">        >>> from einops import einsum</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t"><span class="str">        >>> import torch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t"><span class="str">        >>> import pandas as pd</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M", device="cpu")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t"><span class="str">        >>> prompt = "Why did the chicken cross the"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t"><span class="str">        >>> answer = " road"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t"><span class="str">        >>> logits, cache = model.run_with_cache("Why did the chicken cross the")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t"><span class="str">        >>> answer_token = model.to_single_token(answer)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t"><span class="str">        >>> print(answer_token)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t"><span class="str">        2975</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t"><span class="str">        >>> accum_resid, labels = cache.accumulated_resid(return_labels=True, apply_ln=True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t"><span class="str">        >>> last_token_accum = accum_resid[:, 0, -1, :]  # layer, batch, pos, d_model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t"><span class="str">        >>> print(last_token_accum.shape)  # layer, d_model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t"><span class="str">        torch.Size([9, 64])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t"><span class="str">        >>> W_U = model.W_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t"><span class="str">        >>> print(W_U.shape)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t"><span class="str">        torch.Size([64, 50257])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t"><span class="str">        >>> layers_unembedded = einsum(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t"><span class="str">        ...         last_token_accum,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t"><span class="str">        ...         W_U,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t"><span class="str">        ...         "layer d_model, d_model d_vocab -> layer d_vocab"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t"><span class="str">        ...     )</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t"><span class="str">        >>> print(layers_unembedded.shape)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t"><span class="str">        torch.Size([9, 50257])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t"><span class="str">        >>> # Get the rank of the correct answer by layer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t"><span class="str">        >>> sorted_indices = torch.argsort(layers_unembedded, dim=1, descending=True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t"><span class="str">        >>> rank_answer = (sorted_indices == 2975).nonzero(as_tuple=True)[1]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t"><span class="str">        >>> print(pd.Series(rank_answer, index=labels))</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t"><span class="str">        0_pre         4442</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t"><span class="str">        1_pre          382</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t"><span class="str">        2_pre          982</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t"><span class="str">        3_pre         1160</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t"><span class="str">        4_pre          408</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t"><span class="str">        5_pre          145</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t"><span class="str">        6_pre           78</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t"><span class="str">        7_pre          387</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t"><span class="str">        final_post       6</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t"><span class="str">        dtype: int64</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t"><span class="str">                The layer to take components up to - by default includes resid_pre for that layer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t"><span class="str">                and excludes resid_mid and resid_post for that layer. If set as `n_layers`, `-1` or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t"><span class="str">                `None` it will return all residual streams, including the final one (i.e.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t"><span class="str">                immediately pre logits). The indices are taken such that this gives the accumulated</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t"><span class="str">                streams up to the input to layer l.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t"><span class="str">            incl_mid:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t"><span class="str">                Whether to return `resid_mid` for all previous layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t"><span class="str">                Whether to include resid_mid for the current layer. This essentially gives the MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t"><span class="str">                input rather than the attention input.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t"><span class="str">                Whether to return a list of labels for the residual stream components. Useful for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t"><span class="str">                labelling graphs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t"><span class="str">            A tensor of the accumulated residual streams. If `return_labels` is True, also returns a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t"><span class="str">            list of labels for the components (as a tuple in the form `(components, labels)`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span> <span class="op">+</span> <span class="num">1</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">            <span class="key">if</span> <span class="nam">l</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"final_post"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">                <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">            <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_pre"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_pre"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span><span class="nam">incl_mid</span> <span class="key">and</span> <span class="nam">l</span> <span class="op">&lt;</span> <span class="nam">layer</span><span class="op">)</span> <span class="key">or</span> <span class="op">(</span><span class="nam">mlp_input</span> <span class="key">and</span> <span class="nam">l</span> <span class="op">==</span> <span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_mid"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_mid"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">c</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span> <span class="key">for</span> <span class="nam">c</span> <span class="key">in</span> <span class="nam">components_list</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">    <span class="key">def</span> <span class="nam">logit_attrs</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="nam">residual_stack</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">            <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch position"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="nam">incorrect_tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">            <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">                <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">                <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">                <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch position"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">        <span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims_out"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">        <span class="str">"""Logit Attributions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t"><span class="str">        Takes a residual stack (typically the residual stream decomposed by components), and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t"><span class="str">        calculates how much each item in the stack "contributes" to specific tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t"><span class="str">        It does this by:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t"><span class="str">            1. Getting the residual directions of the tokens (i.e. reversing the unembed)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t"><span class="str">            2. Taking the dot product of each item in the residual stack, with the token residual</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t"><span class="str">               directions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t"><span class="str">        Note that if incorrect tokens are provided, it instead takes the difference between the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t"><span class="str">        correct and incorrect tokens (to calculate the residual directions). This is useful as</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t"><span class="str">        sometimes we want to know e.g. which components are most responsible for selecting the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t"><span class="str">        correct token rather than an incorrect one. For example in the `Interpretability in the Wild</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t"><span class="str">        paper &lt;https://arxiv.org/abs/2211.00593>` prompts such as "John and Mary went to the shops,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t"><span class="str">        John gave a bag to" were investigated, and it was therefore useful to calculate attribution</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t"><span class="str">        for the :math:`\\text{Mary} - \\text{John}` residual direction.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t"><span class="str">        Warning:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t"><span class="str">        Choosing the correct `tokens` and `incorrect_tokens` is both important and difficult. When</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t"><span class="str">        investigating specific components it's also useful to look at it's impact on all tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t"><span class="str">        (i.e. :math:`\\text{final_ln}(\\text{residual_stack_item}) W_U`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t"><span class="str">            residual_stack:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t"><span class="str">                Stack of components of residual stream to get logit attributions for.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t"><span class="str">            tokens:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t"><span class="str">                Tokens to compute logit attributions on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t"><span class="str">            incorrect_tokens:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t"><span class="str">                If provided, compute attributions on logit difference between tokens and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t"><span class="str">                incorrect_tokens. Must have the same shape as tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t"><span class="str">                The slice to apply layer norm scaling on. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t"><span class="str">                The slice to take on the batch dimension during layer norm scaling. Defaults to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t"><span class="str">                None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t"><span class="str">            has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t"><span class="str">                Whether residual_stack has a batch dimension. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t"><span class="str">            A tensor of the logit attributions or logit difference attributions if incorrect_tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t"><span class="str">            was provided.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">        <span class="nam">logit_directions</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">tokens_to_residual_directions</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">        <span class="key">if</span> <span class="nam">incorrect_tokens</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">                <span class="nam">incorrect_tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">                <span class="nam">incorrect_tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">as_tensor</span><span class="op">(</span><span class="nam">incorrect_tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">            <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span> <span class="op">!=</span> <span class="nam">incorrect_tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">                    <span class="str">f"tokens and incorrect_tokens must have the same shape! \</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t"><span class="str">                        (tokens.shape={tokens.shape}, \</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t"><span class="str">                        incorrect_tokens.shape={incorrect_tokens.shape})"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">            <span class="com"># If incorrect_tokens was provided, take the logit difference</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">            <span class="nam">logit_directions</span> <span class="op">=</span> <span class="nam">logit_directions</span> <span class="op">-</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">tokens_to_residual_directions</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">                <span class="nam">incorrect_tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">        <span class="nam">scaled_residual_stack</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">            <span class="nam">residual_stack</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">            <span class="nam">layer</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">            <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">            <span class="nam">batch_slice</span><span class="op">=</span><span class="nam">batch_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">            <span class="nam">has_batch_dim</span><span class="op">=</span><span class="nam">has_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">        <span class="com"># Element-wise multiplication and sum over the d_model dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="nam">logit_attrs</span> <span class="op">=</span> <span class="op">(</span><span class="nam">scaled_residual_stack</span> <span class="op">*</span> <span class="nam">logit_directions</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">        <span class="key">return</span> <span class="nam">logit_attrs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">    <span class="key">def</span> <span class="nam">decompose_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="nam">mode</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"all"</span><span class="op">,</span> <span class="str">"mlp"</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"all"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">        <span class="nam">incl_embeds</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="str">"""Decompose the Residual Stream.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t"><span class="str">        Decomposes the residual stream input to layer L into a stack of the output of previous</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t"><span class="str">        layers. The sum of these is the input to layer L (plus embedding and pos embedding). This is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t"><span class="str">        useful for attributing model behaviour to different components of the residual stream</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t"><span class="str">                The layer to take components up to - by default includes</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">                resid_pre for that layer and excludes resid_mid and resid_post for that layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t"><span class="str">                layer==n_layers means to return all layer outputs incl in the final layer, layer==0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">                means just embed and pos_embed. The indices are taken such that this gives the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t"><span class="str">                accumulated streams up to the input to layer l</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t"><span class="str">                Whether to include attn_out for the current</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t"><span class="str">                layer - essentially decomposing the residual stream that's input to the MLP input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">                rather than the Attn input.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t"><span class="str">            mode:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">                Values are "all", "mlp" or "attn". "all" returns all</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t"><span class="str">                components, "mlp" returns only the MLP components, and "attn" returns only the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t"><span class="str">                attention components. Defaults to "all".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t"><span class="str">                Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t"><span class="str">            incl_embeds:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t"><span class="str">                Whether to include embed &amp; pos_embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t"><span class="str">                Whether to return a list of labels for the residual stream components.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t"><span class="str">                Useful for labelling graphs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t"><span class="str">            A tensor of the accumulated residual streams. If `return_labels` is True, also returns</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t"><span class="str">            a list of labels for the components (as a tuple in the form `(components, labels)`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">        <span class="nam">incl_attn</span> <span class="op">=</span> <span class="nam">mode</span> <span class="op">!=</span> <span class="str">"mlp"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">        <span class="nam">incl_mlp</span> <span class="op">=</span> <span class="nam">mode</span> <span class="op">!=</span> <span class="str">"attn"</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">        <span class="key">if</span> <span class="nam">incl_embeds</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">625&#x202F;&#x219B;&#x202F;628</span><span class="annotate long">line 625 didn't jump to line 628, because the condition on line 625 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">                <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">628&#x202F;&#x219B;&#x202F;632</span><span class="annotate long">line 628 didn't jump to line 632, because the condition on line 628 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_pos_embed"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"pos_embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_attn</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"attn_out"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_attn_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">                <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"mlp_out"</span><span class="op">,</span> <span class="nam">l</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{l}_mlp_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span> <span class="key">and</span> <span class="nam">incl_attn</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">            <span class="nam">components_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"attn_out"</span><span class="op">,</span> <span class="nam">layer</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">f"{layer}_attn_out"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">        <span class="nam">components_list</span> <span class="op">=</span> <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">c</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span> <span class="key">for</span> <span class="nam">c</span> <span class="key">in</span> <span class="nam">components_list</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">    <span class="key">def</span> <span class="nam">compute_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">        <span class="str">"""Compute Head Results.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t"><span class="str">        Computes and caches the results for each attention head, ie the amount contributed to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t"><span class="str">        residual stream from that head. attn_out for a layer is the sum of head results plus b_O.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t"><span class="str">        Intended use is to enable use_attn_results when running and caching the model, but this can</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t"><span class="str">        be useful if you forget.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">        <span class="key">if</span> <span class="str">"blocks.0.attn.hook_result"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Tried to compute head results when they were already cached"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">            <span class="key">return</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">            <span class="com"># Note that we haven't enabled set item on this object so we need to edit the underlying</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">            <span class="com"># cache_dict directly.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">            <span class="com"># Add singleton dimension to match W_O's shape for broadcasting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">            <span class="nam">z</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">                <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"z"</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">)</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">                <span class="str">"... head_index d_head -> ... head_index d_head 1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">            <span class="com"># Element-wise multiplication of z and W_O (with shape [head_index, d_head, d_model])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">            <span class="nam">result</span> <span class="op">=</span> <span class="nam">z</span> <span class="op">*</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">            <span class="com"># Sum over d_head to get the contribution of each head to the residual stream</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.hook_result"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">result</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="nam">incl_remainder</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="str">"""Stack Head Results.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t"><span class="str">        Returns a stack of all head results (ie residual stream contribution) up to layer L. A good</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t"><span class="str">        way to decompose the outputs of attention layers into attribution by specific heads. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t"><span class="str">        that the num_components axis has length layer x n_heads ((layer head_index) in einops</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t"><span class="str">        notation).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t"><span class="str">                Layer index - heads at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t"><span class="str">                in [1, n_layers-1], or any of (n_layers, -1, None), which all mean the final layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t"><span class="str">                Whether to also return a list of labels of the form "L0H0" for the heads.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t"><span class="str">                A slice object to apply to the pos dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">)</span>  <span class="com"># mypy can't seem to infer this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">        <span class="key">if</span> <span class="str">"blocks.0.attn.hook_result"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cache_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">            <span class="nam">print</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">                <span class="str">"Tried to stack head results when they weren't cached. Computing head results now"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">compute_head_results</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">        <span class="nam">components</span><span class="op">:</span> <span class="nam">Any</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t">            <span class="com"># Note that this has shape batch x pos x head_index x d_model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"result"</span><span class="op">,</span> <span class="nam">l</span><span class="op">,</span> <span class="str">"attn"</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">3</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t">        <span class="key">if</span> <span class="nam">components</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">                <span class="str">"... concat_head_index d_model -> concat_head_index ... d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">                <span class="nam">remainder</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">                    <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">                <span class="op">)</span> <span class="op">-</span> <span class="nam">components</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">                <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">components</span><span class="op">,</span> <span class="nam">remainder</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">        <span class="key">elif</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">            <span class="com"># There are no components, so the remainder is the entire thing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">                <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">            <span class="com"># If this is called with layer 0, we return an empty tensor of the right shape to be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">            <span class="com"># stacked correctly. This uses the shape of hook_embed, which is pretty janky since it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">            <span class="com"># assumes embed is in the cache. But it's hard to explicitly code the shape, since it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="com"># depends on the pos slice, whether we have a batch dim, etc. And it's pretty messy!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">                <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">                <span class="op">*</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">                <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_activation</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">        <span class="nam">activation_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">        <span class="nam">sublayer_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"layers_covered ..."</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">        <span class="str">"""Stack Activations.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t"><span class="str">        Flexible way to stack activations with a given name.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t"><span class="str">            activation_name:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t"><span class="str">                The name of the activation to be stacked</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t"><span class="str">                'Layer index - heads' at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t"><span class="str">                in [1, n_layers-1], or any of (n_layers, -1, None), which all mean the final layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t"><span class="str">            sublayer_type:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t"><span class="str">                The sub layer type of the activation, passed to utils.get_act_name. Can normally be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t"><span class="str">                inferred.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">        <span class="nam">neuron_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"*batch_and_pos_dims num_neurons d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="str">"""Get Neuron Results.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t"><span class="str">        Get the results of for neurons in a specific layer (i.e, how much each neuron contributes to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t"><span class="str">        the residual stream). Does it for the subset of neurons specified by neuron_slice, defaults</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t"><span class="str">        to all of them. Does *not* cache these because it's expensive in space and cheap to compute.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="nam">activation_name</span><span class="op">,</span> <span class="nam">l</span><span class="op">,</span> <span class="nam">sublayer_type</span><span class="op">)</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">        <span class="nam">neuron_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"*batch_and_pos_dims num_neurons d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">        <span class="str">"""Get Neuron Results.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t"><span class="str">                Layer index.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t"><span class="str">            neuron_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t"><span class="str">                Slice of the neuron.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t"><span class="str">                Slice of the positions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t"><span class="str">            Tensor of the results.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">            <span class="nam">neuron_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">        <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"post"</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="str">"mlp"</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">        <span class="nam">W_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">        <span class="key">if</span> <span class="nam">pos_slice</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">827&#x202F;&#x219B;&#x202F;831</span><span class="annotate long">line 827 didn't jump to line 831, because the condition on line 827 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">            <span class="com"># Note - order is important, as Slice.apply *may* collapse a dimension, so this ensures</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">            <span class="com"># that position dimension is -2 when we apply position slice</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">            <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">neuron_acts</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">        <span class="key">if</span> <span class="nam">neuron_slice</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">831&#x202F;&#x219B;&#x202F;834</span><span class="annotate long">line 831 didn't jump to line 834, because the condition on line 831 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">neuron_acts</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="nam">W_out</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">W_out</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">        <span class="key">return</span> <span class="nam">neuron_acts</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">        <span class="nam">neuron_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">        <span class="nam">incl_remainder</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">        <span class="str">"""Stack Neuron Results</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t"><span class="str">        Returns a stack of all neuron results (ie residual stream contribution) up to layer L - ie</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t"><span class="str">        the amount each individual neuron contributes to the residual stream. Also returns a list of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t"><span class="str">        labels of the form "L0N0" for the neurons. A good way to decompose the outputs of MLP layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t"><span class="str">        into attribution by specific neurons.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t"><span class="str">        Note that doing this for all neurons is SUPER expensive on GPU memory and only works for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t"><span class="str">        small models or short inputs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t"><span class="str">                Layer index - heads at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t"><span class="str">                in [1, n_layers]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t"><span class="str">                Slice of the positions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t"><span class="str">            neuron_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t"><span class="str">                Slice of the neurons.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t"><span class="str">                Whether to also return a list of labels of the form "L0H0" for the heads.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t"><span class="str">        Get the results of for neurons in a specific layer (i.e, how much each neuron contributes to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t"><span class="str">        the residual stream). Does it for the subset of neurons specified by neuron_slice, defaults</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t"><span class="str">        to all of them. Does *not* cache these because it's expensive in space and cheap to compute.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t"><span class="str">                Layer index.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t"><span class="str">            neuron_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t"><span class="str">                Slice of the neuron.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t"><span class="str">                Slice of the positions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t"><span class="str">            Tensor of the results.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">            <span class="nam">neuron_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">        <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"post"</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="str">"mlp"</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">        <span class="nam">W_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">        <span class="key">if</span> <span class="nam">pos_slice</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">831&#x202F;&#x219B;&#x202F;835</span><span class="annotate long">line 831 didn't jump to line 835, because the condition on line 831 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="com"># Note - order is important, as Slice.apply *may* collapse a dimension, so this ensures</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="com"># that position dimension is -2 when we apply position slice</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">            <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">neuron_acts</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">        <span class="key">if</span> <span class="nam">neuron_slice</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">835&#x202F;&#x219B;&#x202F;838</span><span class="annotate long">line 835 didn't jump to line 838, because the condition on line 835 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">            <span class="nam">neuron_acts</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">neuron_acts</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">            <span class="nam">W_out</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">W_out</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">        <span class="key">return</span> <span class="nam">neuron_acts</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">    <span class="key">def</span> <span class="nam">stack_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">        <span class="nam">neuron_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">        <span class="nam">incl_remainder</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">        <span class="str">"""Stack Neuron Results</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t"><span class="str">        Returns a stack of all neuron results (ie residual stream contribution) up to layer L - ie</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t"><span class="str">        the amount each individual neuron contributes to the residual stream. Also returns a list of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t"><span class="str">        labels of the form "L0N0" for the neurons. A good way to decompose the outputs of MLP layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t"><span class="str">        into attribution by specific neurons.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t"><span class="str">        Note that doing this for all neurons is SUPER expensive on GPU memory and only works for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t"><span class="str">        small models or short inputs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t"><span class="str">                Layer index - heads at all layers strictly before this are included. layer must be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t"><span class="str">                in [1, n_layers]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t"><span class="str">                Slice of the positions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t"><span class="str">            neuron_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t"><span class="str">                Slice of the neurons.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t"><span class="str">                Whether to also return a list of labels of the form "L0H0" for the heads.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t"><span class="str">            incl_remainder:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t"><span class="str">                Whether to return a final term which is "the rest of the residual stream".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">        <span class="nam">components</span><span class="op">:</span> <span class="nam">Any</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>  <span class="com"># TODO: fix typing properly</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">            <span class="nam">neuron_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">        <span class="nam">neuron_labels</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">|</span> <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span><span class="op">)</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">        <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">neuron_labels</span><span class="op">)</span> <span class="op">==</span> <span class="nam">int</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">889&#x202F;&#x219B;&#x202F;890</span><span class="annotate long">line 889 didn't jump to line 890, because the condition on line 889 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">            <span class="nam">neuron_labels</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">array</span><span class="op">(</span><span class="op">[</span><span class="nam">neuron_labels</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">            <span class="com"># Note that this has shape batch x pos x head_index x d_model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">get_neuron_results</span><span class="op">(</span><span class="nam">l</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">neuron_slice</span><span class="op">=</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="op">[</span><span class="str">f"L{l}N{h}"</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">neuron_labels</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">        <span class="key">if</span> <span class="nam">components</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">                <span class="str">"... concat_neuron_index d_model -> concat_neuron_index ... d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">                <span class="nam">remainder</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">                    <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">                <span class="op">)</span> <span class="op">-</span> <span class="nam">components</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">                <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">components</span><span class="op">,</span> <span class="nam">remainder</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">        <span class="key">elif</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">                <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">            <span class="com"># Returning empty, give it the right shape to stack properly</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">                <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">                <span class="op">*</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">                <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">        <span class="nam">residual_stack</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">        <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">        <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims_out d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">        <span class="str">"""Apply Layer Norm to a Stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t"><span class="str">        Takes a stack of components of the residual stream (eg outputs of decompose_resid or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t"><span class="str">        accumulated_resid), treats them as the input to a specific layer, and applies the layer norm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t"><span class="str">        scaling of that layer to them, using the cached scale factors - simulating what that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t"><span class="str">        component of the residual stream contributes to that layer's input.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t"><span class="str">        The layernorm scale is global across the entire residual stream for each layer, batch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t"><span class="str">        element and position, which is why we need to use the cached scale factors rather than just</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t"><span class="str">        applying a new LayerNorm.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">        <span class="nam">components</span><span class="op">:</span> <span class="nam">Any</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>  <span class="com"># TODO: fix typing properly</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">            <span class="nam">neuron_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">        <span class="nam">neuron_labels</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">|</span> <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span> <span class="op">=</span> <span class="nam">neuron_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_mlp</span><span class="op">)</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">        <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">neuron_labels</span><span class="op">)</span> <span class="op">==</span> <span class="nam">int</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">893&#x202F;&#x219B;&#x202F;894</span><span class="annotate long">line 893 didn't jump to line 894, because the condition on line 893 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">            <span class="nam">neuron_labels</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">array</span><span class="op">(</span><span class="op">[</span><span class="nam">neuron_labels</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">            <span class="com"># Note that this has shape batch x pos x head_index x d_model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">get_neuron_results</span><span class="op">(</span><span class="nam">l</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">neuron_slice</span><span class="op">=</span><span class="nam">neuron_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="op">[</span><span class="str">f"L{l}N{h}"</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">neuron_labels</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">        <span class="key">if</span> <span class="nam">components</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">                <span class="nam">components</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">                <span class="str">"... concat_neuron_index d_model -> concat_neuron_index ... d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">                <span class="nam">remainder</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">                    <span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">                <span class="op">)</span> <span class="op">-</span> <span class="nam">components</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">                <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">components</span><span class="op">,</span> <span class="nam">remainder</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">        <span class="key">elif</span> <span class="nam">incl_remainder</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">                <span class="op">[</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="op">(</span><span class="str">"resid_post"</span><span class="op">,</span> <span class="nam">layer</span> <span class="op">-</span> <span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"remainder"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">            <span class="com"># Returning empty, give it the right shape to stack properly</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">                <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">                <span class="op">*</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"hook_embed"</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">                <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">            <span class="nam">components</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">            <span class="key">return</span> <span class="nam">components</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">        <span class="nam">residual_stack</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">        <span class="nam">batch_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t">        <span class="nam">has_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims_out d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t">        <span class="str">"""Apply Layer Norm to a Stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t"><span class="str">        Takes a stack of components of the residual stream (eg outputs of decompose_resid or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t"><span class="str">        accumulated_resid), treats them as the input to a specific layer, and applies the layer norm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t"><span class="str">        scaling of that layer to them, using the cached scale factors - simulating what that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t"><span class="str">        component of the residual stream contributes to that layer's input.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t"><span class="str">        If the model does not use LayerNorm or RMSNorm, it returns the residual stack unchanged.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t"><span class="str">            residual_stack:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t"><span class="str">                A tensor, whose final dimension is d_model. The other trailing dimensions are</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t"><span class="str">                assumed to be the same as the stored hook_scale - which may or may not include batch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t"><span class="str">                or position dimensions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t"><span class="str">                The layer we're taking the input to. In [0, n_layers], n_layers means the unembed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t"><span class="str">                None maps to the n_layers case, ie the unembed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t"><span class="str">                Whether the input is to the MLP or attn (ie ln2 vs ln1). Defaults to False, ie ln1.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t"><span class="str">                If layer==n_layers, must be False, and we use ln_final</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t"><span class="str">                The slice to take of positions, if residual_stack is not over the full context, None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t"><span class="str">                means do nothing. It is assumed that pos_slice has already been applied to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t"><span class="str">                residual_stack, and this is only applied to the scale. See utils.Slice for details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t"><span class="str">                Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t"><span class="str">                The slice to take on the batch dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t"><span class="str">            has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t"><span class="str">                Whether residual_stack has a batch dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">,</span> <span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">975&#x202F;&#x219B;&#x202F;977</span><span class="annotate long">line 975 didn't jump to line 977, because the condition on line 975 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="com"># The model does not use LayerNorm, so we don't need to do anything.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t"><span class="str">        The layernorm scale is global across the entire residual stream for each layer, batch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t"><span class="str">        element and position, which is why we need to use the cached scale factors rather than just</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t"><span class="str">        applying a new LayerNorm.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t"><span class="str">        If the model does not use LayerNorm or RMSNorm, it returns the residual stack unchanged.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t"><span class="str">            residual_stack:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t"><span class="str">                A tensor, whose final dimension is d_model. The other trailing dimensions are</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t"><span class="str">                assumed to be the same as the stored hook_scale - which may or may not include batch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t"><span class="str">                or position dimensions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t"><span class="str">                The layer we're taking the input to. In [0, n_layers], n_layers means the unembed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t"><span class="str">                None maps to the n_layers case, ie the unembed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t"><span class="str">                Whether the input is to the MLP or attn (ie ln2 vs ln1). Defaults to False, ie ln1.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t"><span class="str">                If layer==n_layers, must be False, and we use ln_final</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t"><span class="str">                The slice to take of positions, if residual_stack is not over the full context, None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t"><span class="str">                means do nothing. It is assumed that pos_slice has already been applied to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t"><span class="str">                residual_stack, and this is only applied to the scale. See utils.Slice for details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t"><span class="str">                Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t"><span class="str">            batch_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t"><span class="str">                The slice to take on the batch dimension. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t"><span class="str">            has_batch_dim:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t"><span class="str">                Whether residual_stack has a batch dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">,</span> <span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">979&#x202F;&#x219B;&#x202F;981</span><span class="annotate long">line 979 didn't jump to line 981, because the condition on line 979 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">            <span class="com"># The model does not use LayerNorm, so we don't need to do anything.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="nam">batch_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">batch_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">        <span class="key">if</span> <span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">            <span class="com"># Apply batch slice to the stack</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">        <span class="com"># Center the stack onlny if the model uses LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">992&#x202F;&#x219B;&#x202F;995</span><span class="annotate long">line 992 didn't jump to line 995, because the condition on line 992 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">residual_stack</span> <span class="op">-</span> <span class="nam">residual_stack</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">        <span class="key">if</span> <span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">            <span class="com"># Apply batch slice to the stack</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="key">or</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="str">"ln_final.hook_scale"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="str">f"blocks.{layer}.ln{2 if mlp_input else 1}.hook_scale"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">        <span class="com"># The shape of scale is [batch, position, 1] or [position, 1] - final dimension is a dummy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">        <span class="com"># thing to get broadcoasting to work nicely.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">        <span class="nam">scale</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">        <span class="com"># Center the stack onlny if the model uses LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">996&#x202F;&#x219B;&#x202F;999</span><span class="annotate long">line 996 didn't jump to line 999, because the condition on line 996 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">residual_stack</span> <span class="op">-</span> <span class="nam">residual_stack</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="key">or</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="str">"ln_final.hook_scale"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="str">f"blocks.{layer}.ln{2 if mlp_input else 1}.hook_scale"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">self</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1005&#x202F;&#x219B;&#x202F;1009</span><span class="annotate long">line 1005 didn't jump to line 1009, because the condition on line 1005 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">            <span class="com"># Apply batch slice to the scale</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">        <span class="com"># The shape of scale is [batch, position, 1] or [position, 1] - final dimension is a dummy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">        <span class="com"># thing to get broadcoasting to work nicely.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">        <span class="nam">scale</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">        <span class="key">return</span> <span class="nam">residual_stack</span> <span class="op">/</span> <span class="nam">scale</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_full_resid_decomposition</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">        <span class="nam">expand_neurons</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">        <span class="str">"""Get the full Residual Decomposition.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t"><span class="str">        Returns the full decomposition of the residual stream into embed, pos_embed, each head</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t"><span class="str">        result, each neuron result, and the accumulated biases. We break down the residual stream</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t"><span class="str">        that is input into some layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1009&#x202F;&#x219B;&#x202F;1013</span><span class="annotate long">line 1009 didn't jump to line 1013, because the condition on line 1009 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">            <span class="com"># Apply batch slice to the scale</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">            <span class="nam">scale</span> <span class="op">=</span> <span class="nam">batch_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">        <span class="key">return</span> <span class="nam">residual_stack</span> <span class="op">/</span> <span class="nam">scale</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_full_resid_decomposition</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">        <span class="nam">layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">        <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">        <span class="nam">expand_neurons</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">        <span class="nam">apply_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">        <span class="nam">return_labels</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"num_components *batch_and_pos_dims d_model"</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">        <span class="str">"""Get the full Residual Decomposition.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t"><span class="str">                The layer we're inputting into. layer is in [0, n_layers], if layer==n_layers (or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t"><span class="str">                None) we're inputting into the unembed (the entire stream), if layer==0 then it's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t"><span class="str">                just embed and pos_embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t"><span class="str">                Are we inputting to the MLP in that layer or the attn? Must be False for final</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t"><span class="str">                layer, since that's the unembed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t"><span class="str">            expand_neurons:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t"><span class="str">                Whether to expand the MLP outputs to give every neuron's result or just return the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t"><span class="str">                MLP layer outputs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t"><span class="str">                Slice of the positions to take.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t"><span class="str">                Whether to return the labels.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">        <span class="key">assert</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">        <span class="nam">head_stack</span><span class="op">,</span> <span class="nam">head_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">stack_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">            <span class="nam">layer</span> <span class="op">+</span> <span class="op">(</span><span class="num">1</span> <span class="key">if</span> <span class="nam">mlp_input</span> <span class="key">else</span> <span class="num">0</span><span class="op">)</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="nam">head_labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="op">[</span><span class="nam">head_stack</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span> <span class="key">and</span> <span class="nam">layer</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">            <span class="key">if</span> <span class="nam">expand_neurons</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">                <span class="nam">neuron_stack</span><span class="op">,</span> <span class="nam">neuron_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">stack_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">                    <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="nam">neuron_labels</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">                <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">neuron_stack</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">                <span class="com"># Get the stack of just the MLP outputs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">                <span class="com"># mlp_input included for completeness, but it doesn't actually matter, since it's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">                <span class="com"># just for MLP outputs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">                <span class="nam">mlp_stack</span><span class="op">,</span> <span class="nam">mlp_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">decompose_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">                    <span class="nam">layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">                    <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">                    <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">                    <span class="nam">incl_embeds</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">                    <span class="nam">mode</span><span class="op">=</span><span class="str">"mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">                    <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="nam">mlp_labels</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">                <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">mlp_stack</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1081&#x202F;&#x219B;&#x202F;1084</span><span class="annotate long">line 1081 didn't jump to line 1084, because the condition on line 1081 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"embed"</span><span class="op">]</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1084&#x202F;&#x219B;&#x202F;1088</span><span class="annotate long">line 1084 didn't jump to line 1088, because the condition on line 1084 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"pos_embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"pos_embed"</span><span class="op">]</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">        <span class="com"># If we didn't expand the neurons, the MLP biases are already included in the MLP outputs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">        <span class="nam">bias</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">accumulated_bias</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="nam">expand_neurons</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">        <span class="nam">bias</span> <span class="op">=</span> <span class="nam">bias</span><span class="op">.</span><span class="nam">expand</span><span class="op">(</span><span class="op">(</span><span class="num">1</span><span class="op">,</span><span class="op">)</span> <span class="op">+</span> <span class="nam">head_stack</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">        <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"bias"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">        <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">bias</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">        <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">                <span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t"><span class="str">        Returns the full decomposition of the residual stream into embed, pos_embed, each head</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t"><span class="str">        result, each neuron result, and the accumulated biases. We break down the residual stream</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t"><span class="str">        that is input into some layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t"><span class="str">            layer:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t"><span class="str">                The layer we're inputting into. layer is in [0, n_layers], if layer==n_layers (or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t"><span class="str">                None) we're inputting into the unembed (the entire stream), if layer==0 then it's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t"><span class="str">                just embed and pos_embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t"><span class="str">            mlp_input:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t"><span class="str">                Are we inputting to the MLP in that layer or the attn? Must be False for final</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t"><span class="str">                layer, since that's the unembed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t"><span class="str">            expand_neurons:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t"><span class="str">                Whether to expand the MLP outputs to give every neuron's result or just return the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t"><span class="str">                MLP layer outputs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t"><span class="str">            apply_ln:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t"><span class="str">                Whether to apply LayerNorm to the stack.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t"><span class="str">                Slice of the positions to take.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t"><span class="str">            return_labels:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t"><span class="str">                Whether to return the labels.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">        <span class="key">if</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">layer</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">            <span class="com"># Default to the residual stream immediately pre unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">            <span class="nam">layer</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">        <span class="key">assert</span> <span class="nam">layer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">Slice</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">            <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">        <span class="nam">head_stack</span><span class="op">,</span> <span class="nam">head_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">stack_head_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">            <span class="nam">layer</span> <span class="op">+</span> <span class="op">(</span><span class="num">1</span> <span class="key">if</span> <span class="nam">mlp_input</span> <span class="key">else</span> <span class="num">0</span><span class="op">)</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="nam">head_labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">        <span class="nam">components</span> <span class="op">=</span> <span class="op">[</span><span class="nam">head_stack</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span> <span class="key">and</span> <span class="nam">layer</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="key">if</span> <span class="nam">expand_neurons</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">                <span class="nam">neuron_stack</span><span class="op">,</span> <span class="nam">neuron_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">stack_neuron_results</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">                    <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="nam">neuron_labels</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">                <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">neuron_stack</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">                <span class="com"># Get the stack of just the MLP outputs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">                <span class="com"># mlp_input included for completeness, but it doesn't actually matter, since it's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">                <span class="com"># just for MLP outputs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">                <span class="nam">mlp_stack</span><span class="op">,</span> <span class="nam">mlp_labels</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">decompose_resid</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">                    <span class="nam">layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">                    <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">                    <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">                    <span class="nam">incl_embeds</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">                    <span class="nam">mode</span><span class="op">=</span><span class="str">"mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">                    <span class="nam">return_labels</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="nam">mlp_labels</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">                <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">mlp_stack</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1085&#x202F;&#x219B;&#x202F;1088</span><span class="annotate long">line 1085 didn't jump to line 1088, because the condition on line 1085 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"embed"</span><span class="op">]</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">has_pos_embed</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1088&#x202F;&#x219B;&#x202F;1092</span><span class="annotate long">line 1088 didn't jump to line 1092, because the condition on line 1088 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">            <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"pos_embed"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">            <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">self</span><span class="op">[</span><span class="str">"pos_embed"</span><span class="op">]</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">        <span class="com"># If we didn't expand the neurons, the MLP biases are already included in the MLP outputs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">        <span class="nam">bias</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">model</span><span class="op">.</span><span class="nam">accumulated_bias</span><span class="op">(</span><span class="nam">layer</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="nam">expand_neurons</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">        <span class="nam">bias</span> <span class="op">=</span> <span class="nam">bias</span><span class="op">.</span><span class="nam">expand</span><span class="op">(</span><span class="op">(</span><span class="num">1</span><span class="op">,</span><span class="op">)</span> <span class="op">+</span> <span class="nam">head_stack</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">        <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="str">"bias"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">        <span class="nam">components</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">bias</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">        <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="nam">components</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">        <span class="key">if</span> <span class="nam">apply_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">            <span class="nam">residual_stack</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">apply_ln_to_stack</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t">                <span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">layer</span><span class="op">,</span> <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">=</span><span class="nam">mlp_input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_labels</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span><span class="op">,</span> <span class="nam">labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_stack</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -1192,7 +1196,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
index 6e4e93c31..14d2e7f07 100644
--- a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
+++ b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -365,7 +365,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
index c0f633d5b..90f3ee72e 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -507,7 +507,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
index b9c51481b..f7f95782c 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -461,7 +461,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
index a85cc1fdd..112fab43f 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -463,7 +463,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
index eda632229..6dd470c00 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">738 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">597<span class="text"> run</span></button>
+            <span class="text">740 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">599<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">141<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">90<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -114,2467 +114,2474 @@ <h2>
     <p class="run"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span> <span class="key">as</span> <span class="nam">nn</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span><span class="op">.</span><span class="nam">functional</span> <span class="key">as</span> <span class="nam">F</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t"><span class="key">import</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">auto</span> <span class="key">as</span> <span class="nam">tqdm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t"><span class="key">from</span> <span class="nam">fancy_einsum</span> <span class="key">import</span> <span class="nam">einsum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span><span class="op">,</span> <span class="nam">Int</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="key">from</span> <span class="nam">packaging</span> <span class="key">import</span> <span class="nam">version</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t"><span class="key">from</span> <span class="nam">transformers</span> <span class="key">import</span> <span class="nam">AutoModelForCausalLM</span><span class="op">,</span> <span class="nam">AutoTokenizer</span><span class="op">,</span> <span class="nam">PreTrainedTokenizerBase</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">loading_from_pretrained</span> <span class="key">as</span> <span class="nam">loading</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">as</span> <span class="nam">utils</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">ActivationCache</span> <span class="key">import</span> <span class="nam">ActivationCache</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">components</span> <span class="key">import</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t">    <span class="nam">Embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">    <span class="nam">LayerNorm</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t">    <span class="nam">LayerNormPre</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">    <span class="nam">PosEmbed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">    <span class="nam">RMSNorm</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t">    <span class="nam">RMSNormPre</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">    <span class="nam">TransformerBlock</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">    <span class="nam">Unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">FactoredMatrix</span> <span class="key">import</span> <span class="nam">FactoredMatrix</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">hook_points</span> <span class="key">import</span> <span class="nam">HookedRootModule</span><span class="op">,</span> <span class="nam">HookPoint</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">HookedTransformerConfig</span> <span class="key">import</span> <span class="nam">HookedTransformerConfig</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">loading_from_pretrained</span> <span class="key">import</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="com"># Note - activation cache is used with run_with_cache, past_key_value_caching is used for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="com"># generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">past_key_value_caching</span> <span class="key">import</span> <span class="nam">HookedTransformerKeyValueCache</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utilities</span> <span class="key">import</span> <span class="nam">devices</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t">    <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t">    <span class="nam">init_kaiming_normal_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">    <span class="nam">init_kaiming_uniform_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">    <span class="nam">init_xavier_normal_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t">    <span class="nam">init_xavier_uniform_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="nam">SingleLoss</span> <span class="op">=</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span>  <span class="com"># Type alias for a single element tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="nam">LossPerToken</span> <span class="op">=</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos-1"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="nam">Loss</span> <span class="op">=</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">SingleLoss</span><span class="op">,</span> <span class="nam">LossPerToken</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"><span class="nam">DTYPE_FROM_STRING</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t">    <span class="str">"float32"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t">    <span class="str">"fp32"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">    <span class="str">"float16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">    <span class="str">"fp16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t">    <span class="str">"bfloat16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t">    <span class="str">"bf16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"><span class="nam">T</span> <span class="op">=</span> <span class="nam">TypeVar</span><span class="op">(</span><span class="str">"T"</span><span class="op">,</span> <span class="nam">bound</span><span class="op">=</span><span class="str">"HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span><span class="op">,</span> <span class="nam">Int</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"><span class="key">from</span> <span class="nam">packaging</span> <span class="key">import</span> <span class="nam">version</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="key">from</span> <span class="nam">transformers</span> <span class="key">import</span> <span class="nam">AutoModelForCausalLM</span><span class="op">,</span> <span class="nam">AutoTokenizer</span><span class="op">,</span> <span class="nam">PreTrainedTokenizerBase</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">loading_from_pretrained</span> <span class="key">as</span> <span class="nam">loading</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="key">import</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">as</span> <span class="nam">utils</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">ActivationCache</span> <span class="key">import</span> <span class="nam">ActivationCache</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">components</span> <span class="key">import</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t">    <span class="nam">Embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t">    <span class="nam">LayerNorm</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">    <span class="nam">LayerNormPre</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t">    <span class="nam">PosEmbed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">    <span class="nam">RMSNorm</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">    <span class="nam">RMSNormPre</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t">    <span class="nam">TransformerBlock</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">    <span class="nam">Unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">FactoredMatrix</span> <span class="key">import</span> <span class="nam">FactoredMatrix</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">hook_points</span> <span class="key">import</span> <span class="nam">HookedRootModule</span><span class="op">,</span> <span class="nam">HookPoint</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">HookedTransformerConfig</span> <span class="key">import</span> <span class="nam">HookedTransformerConfig</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">loading_from_pretrained</span> <span class="key">import</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"><span class="com"># Note - activation cache is used with run_with_cache, past_key_value_caching is used for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="com"># generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">past_key_value_caching</span> <span class="key">import</span> <span class="nam">HookedTransformerKeyValueCache</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utilities</span> <span class="key">import</span> <span class="nam">devices</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t">    <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t">    <span class="nam">init_kaiming_normal_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t">    <span class="nam">init_kaiming_uniform_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">    <span class="nam">init_xavier_normal_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">    <span class="nam">init_xavier_uniform_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="nam">SingleLoss</span> <span class="op">=</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span>  <span class="com"># Type alias for a single element tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="nam">LossPerToken</span> <span class="op">=</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos-1"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="nam">Loss</span> <span class="op">=</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">SingleLoss</span><span class="op">,</span> <span class="nam">LossPerToken</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"><span class="nam">DTYPE_FROM_STRING</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t">    <span class="str">"float32"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t">    <span class="str">"fp32"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t">    <span class="str">"float16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">    <span class="str">"fp16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">    <span class="str">"bfloat16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t">    <span class="str">"bf16"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="nam">T</span> <span class="op">=</span> <span class="nam">TypeVar</span><span class="op">(</span><span class="str">"T"</span><span class="op">,</span> <span class="nam">bound</span><span class="op">=</span><span class="str">"HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t"><span class="key">class</span> <span class="nam">Output</span><span class="op">(</span><span class="nam">NamedTuple</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t">    <span class="str">"""Output Named Tuple.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t"><span class="str">    Named tuple object for if we want to output both logits and loss.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t">    <span class="nam">logits</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t">    <span class="nam">loss</span><span class="op">:</span> <span class="nam">Loss</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t"><span class="key">class</span> <span class="nam">Output</span><span class="op">(</span><span class="nam">NamedTuple</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t">    <span class="str">"""Output Named Tuple.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t"><span class="str">    Named tuple object for if we want to output both logits and loss.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t">    <span class="nam">logits</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t">    <span class="nam">loss</span><span class="op">:</span> <span class="nam">Loss</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t"><span class="key">class</span> <span class="nam">HookedTransformer</span><span class="op">(</span><span class="nam">HookedRootModule</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t">    <span class="str">"""Hooked Transformer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"><span class="str">    Implements a full Transformer using the components :doc:`here &lt;transformer_lens.components>`,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t"><span class="str">    with a :class:`transformer_lens.hook_points.HookPoint` on every interesting activation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"><span class="str">    TransformerLens comes loaded with >50 GPT-style models. Typically you initialise it with one of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t"><span class="str">    these via :meth:`from_pretrained`, although it can also be instantiated with randomly</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str">    initialized weights via :meth:`__init__`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str">    Once you've initialized the model, a common next step is to test it can do the task you're</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str">    investigating. This can be done with :func:`transformer_lens.utils.test_prompt`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t">    <span class="nam">ln_final</span><span class="op">:</span> <span class="nam">nn</span><span class="op">.</span><span class="nam">Module</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">        <span class="nam">cfg</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">HookedTransformerConfig</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">PreTrainedTokenizerBase</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">        <span class="str">"""Model initialization.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t"><span class="str">        Note that if you want to load the model from pretrained weights, you should use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t"><span class="str">        :meth:`from_pretrained` instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t"><span class="str">            cfg: The config to use for the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not provided, it is inferred from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t"><span class="str">                `cfg.tokenizer_name` or initialized to `None`. If `None`, then the model cannot be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"><span class="str">                passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in cfg.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t"><span class="str">                device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">        <span class="nam">super</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">__init__</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">134&#x202F;&#x219B;&#x202F;135</span><span class="annotate long">line 134 didn't jump to line 135, because the condition on line 134 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">                <span class="str">"Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">                <span class="str">"pretrained model, use HookedTransformer.from_pretrained() instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">set_tokenizer</span><span class="op">(</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t">            <span class="com"># If we have a tokenizer name, we can load it from HuggingFace</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">146&#x202F;&#x219B;&#x202F;147</span><span class="annotate long">line 146 didn't jump to line 147, because the condition on line 146 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t">                    <span class="str">"%s tokenizer not loaded. Please load manually."</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">                <span class="com"># Hugging Face defaults to use_fast to True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">                <span class="nam">use_fast</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">                <span class="com"># Phi model's fast tokenizer does not support adding a BOS token, use_fast</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">                <span class="com"># should be False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">                <span class="key">if</span> <span class="str">"phi"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">156&#x202F;&#x219B;&#x202F;157</span><span class="annotate long">line 156 didn't jump to line 157, because the condition on line 156 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">                    <span class="nam">use_fast</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">                <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">set_tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">                    <span class="nam">AutoTokenizer</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t">                        <span class="nam">add_bos_token</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">                        <span class="nam">trust_remote_code</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">trust_remote_code</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t">                        <span class="nam">use_fast</span><span class="op">=</span><span class="nam">use_fast</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">                        <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">                    <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">            <span class="com"># If no tokenizer name is provided, we assume we're training on an algorithmic task and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">            <span class="com"># will pass in tokens directly. In this case, we don't need a tokenizer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">!=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if d_vocab is not provided"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">            <span class="key">if</span> <span class="nam">default_padding_side</span> <span class="op">!=</span> <span class="str">"right"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">174&#x202F;&#x219B;&#x202F;175</span><span class="annotate long">line 174 didn't jump to line 175, because the condition on line 174 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">                    <span class="str">"default_padding_side is explictly given but ignored because tokenizer is not set."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span> <span class="op">=</span> <span class="nam">Embed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">PosEmbed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos, d__dictmodel]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_tokens</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_tokens</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span> <span class="op">=</span> <span class="nam">nn</span><span class="op">.</span><span class="nam">ModuleList</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">            <span class="op">[</span><span class="nam">TransformerBlock</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">block_index</span><span class="op">)</span> <span class="key">for</span> <span class="nam">block_index</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">193&#x202F;&#x219B;&#x202F;194</span><span class="annotate long">line 193 didn't jump to line 194, because the condition on line 193 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">195&#x202F;&#x219B;&#x202F;196</span><span class="annotate long">line 195 didn't jump to line 196, because the condition on line 195 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">198&#x202F;&#x219B;&#x202F;199</span><span class="annotate long">line 198 didn't jump to line 199, because the condition on line 198 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LNPre"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">            <span class="com"># We've folded in LayerNorm weights, so just need the center + scale parts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">208&#x202F;&#x219B;&#x202F;212</span><span class="annotate long">line 208 didn't jump to line 212, because the condition on line 208 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">            <span class="com"># If it's None, don't create either layer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Invalid normalization_type passed in %s"</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span> <span class="op">=</span> <span class="nam">Unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">init_weights</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">            <span class="com"># We load the devices in a pipeline manner - the first device gets the embed and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">            <span class="com"># pos_embed layers and the first n_layers // n_devices blocks, the second gets the next</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">            <span class="com"># n_layers // n_devices blocks ... the last gets the last n_layers // n_devices blocks,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">            <span class="com"># the final normalization layer (if it exists) and the unembed layer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">        <span class="com"># Helper variable to store a small (10K-20K) dataset of training data. Empty by default, can</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">        <span class="com"># be loaded with load_sample_training_dataset</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">        <span class="com"># Gives each module a parameter with its name (relative to this root module)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">        <span class="com"># Needed for HookPoints to work</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">setup</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">    <span class="key">def</span> <span class="nam">check_hooks_to_add</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">        <span class="nam">hook_point</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">        <span class="nam">hook_point_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">        <span class="nam">hook</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">        <span class="nam">dir</span><span class="op">=</span><span class="str">"fwd"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">        <span class="nam">is_permanent</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">        <span class="nam">prepend</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"attn.hook_result"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_attn_result_hook is False"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="op">(</span><span class="str">"hook_q_input"</span><span class="op">,</span> <span class="str">"hook_k_input"</span><span class="op">,</span> <span class="str">"hook_v_input"</span><span class="op">)</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_split_qkv_input is False"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"mlp_in"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_hook_mlp_in is False"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"attn_in"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_attn_in is False"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">    <span class="key">def</span> <span class="nam">input_to_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>  <span class="com"># residual</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>  <span class="com"># tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>  <span class="com"># shortformer_pos_embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>  <span class="com"># attention_mask [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">        <span class="str">"""Convert input to first residual stream.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t"><span class="str">            input (Union[str, List[str], Int[torch.Tensor, "batch pos"]]): The input to the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t"><span class="str">            padding_side ([Literal["left", "right"], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t"><span class="str">                multiple strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t"><span class="str">            past_kv_cache (HookedTransformerKeyValueCache, optional): If passed, we're doing caching</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t"><span class="str">                and attention_mask will be stored in the cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span> <span class="key">or</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">            <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">            <span class="com"># This is only intended to support passing in a single string</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">295&#x202F;&#x219B;&#x202F;297</span><span class="annotate long">line 295 didn't jump to line 297, because the condition on line 295 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">            <span class="com"># If tokens are a rank 1 tensor, add a dummy batch dimension to avoid things breaking.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">[</span><span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span><span class="op">.</span><span class="nam">type</span> <span class="op">!=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">            <span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">padding_side</span> <span class="op">==</span> <span class="str">"left"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">            <span class="key">or</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">            <span class="key">or</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">            <span class="com"># This means we need to have an explicit attention mask.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">            <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">                <span class="com"># If the padding side is left or we are using caching, we need to compute the attention</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">                <span class="com"># mask for the adjustment of absolute positional embeddings and attention masking so</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">                <span class="com"># that pad tokens are not attended.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">                <span class="key">if</span> <span class="nam">prepend_bos</span> <span class="key">is</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">                    <span class="nam">prepend_bos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">                <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_attention_mask</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">            <span class="key">assert</span> <span class="nam">attention_mask</span><span class="op">.</span><span class="nam">shape</span> <span class="op">==</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">                <span class="str">f"Attention mask shape {attention_mask.shape} does not match tokens shape "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">                <span class="str">f"{tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">            <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">attention_mask</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">            <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">                <span class="com"># past_kv_cache is not None, so we're doing caching.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">                <span class="com"># We need to extend the previous attention_mask.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">                <span class="com"># Update the past_kv_cache with the new attention_mask (unless it's frozen)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">                <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">past_kv_cache</span><span class="op">.</span><span class="nam">append_attention_mask</span><span class="op">(</span><span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">            <span class="com"># We separate this case from for computational efficiency.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">            <span class="nam">attention_mask</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">        <span class="com"># If we're doing caching, then we reuse keys and values from previous runs, as that's the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="com"># only way that past activations will affect the final logits. The cache contains those so</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="com"># we don't need to recompute them. This is useful for generating text. As we have absolute</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="com"># positional encodings, to implement this we have a `pos_offset` variable, defaulting to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">        <span class="com"># zero, which says to offset which positional encodings are used (cached keys and values</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">        <span class="com"># were calculated with their own positional encodings).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">        <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">            <span class="nam">pos_offset</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">            <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">                <span class="nam">cached_batch_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">                <span class="nam">cache_ctx_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">                <span class="nam">num_heads_in_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">                <span class="nam">d_head_in_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">            <span class="op">)</span> <span class="op">=</span> <span class="nam">past_kv_cache</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">.</span><span class="nam">past_keys</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">            <span class="key">assert</span> <span class="nam">cached_batch_size</span> <span class="op">==</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">346&#x202F;&#x219B;&#x202F;349</span><span class="annotate long">line 346 didn't jump to line 349, because the condition on line 346 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">                <span class="key">assert</span> <span class="nam">num_heads_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">                <span class="key">assert</span> <span class="nam">num_heads_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">            <span class="key">assert</span> <span class="nam">d_head_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">            <span class="nam">pos_offset</span> <span class="op">=</span> <span class="nam">cache_ctx_length</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_tokens</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_tokens</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">        <span class="nam">embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"standard"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">            <span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span> <span class="op">+</span> <span class="nam">pos_embed</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">            <span class="com"># If we're using shortformer style attention, we don't add the positional embedding to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">            <span class="com"># the residual stream. See HookedTransformerConfig for details</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">            <span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="nam">pos_embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">            <span class="com"># Rotary doesn't use positional embeddings, instead they're applied when dot producting</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">            <span class="com"># keys and queries. See HookedTransformerConfig for details</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"alibi"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">374&#x202F;&#x219B;&#x202F;379</span><span class="annotate long">line 374 didn't jump to line 379, because the condition on line 374 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">            <span class="com"># ALiBi does not add positional embeddings to word embeddings,instead it biases QK attention scores.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">                <span class="str">f"Invalid positional_embedding_type passed in {self.cfg.positional_embedding_type}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">        <span class="key">return</span> <span class="nam">residual</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">shortformer_pos_embed</span><span class="op">,</span> <span class="nam">attention_mask</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"logits"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Loss</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"loss"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Loss</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"both"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="nam">Loss</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="str">"""Forward Pass.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t"><span class="str">        Input is either a batch of tokens ([batch, pos]) or a text string, a string is automatically</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t"><span class="str">        tokenized to a batch of a single element. The prepend_bos flag only applies when inputting a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t"><span class="str">        text string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t"><span class="str">        Note that loss is the standard "predict the next token" cross-entropy loss for GPT-2 style</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t"><span class="str">        language models - if you want a custom loss function, the recommended behaviour is returning</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t"><span class="str">        the logits and then applying your custom loss function.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t"><span class="str">            return_type Optional[str]: The type of output to return. Can be one of: None (return</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t"><span class="str">                nothing, don't calculate logits), 'logits' (return logits), 'loss' (return</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t"><span class="str">                cross-entropy loss), 'both' (return logits and loss).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t"><span class="str">            loss_per_token bool: Whether to return the (next token prediction) loss per token (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t"><span class="str">                or average (False). Average loss is a scalar (averaged over position *and* batch),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t"><span class="str">                per-token loss is a tensor ([batch, position-1]) - position-1 because we're</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t"><span class="str">                predicting the next token, and there's no specified next token for the final token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t"><span class="str">                Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t"><span class="str">            prepend_bos Optional[bool]: Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t"><span class="str">                otherwise. (Even for models not explicitly trained with a prepended BOS token, heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t"><span class="str">                often use the first position as a resting position and accordingly lose information</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t"><span class="str">                from the first token, so this empirically seems to give better results.) Pass True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t"><span class="str">                or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t"><span class="str">            padding_side Optional[Literal["left", "right"]]: Overrides self.tokenizer.padding_side.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t"><span class="str">                Specifies which side to pad on when tokenizing multiple strings of different</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t"><span class="str">                lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t"><span class="str">            start_at_layer Optional[int]: If not None, start the forward pass at the specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t"><span class="str">                layer. Requires input to be the residual stream before the specified layer with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t"><span class="str">                shape [batch, pos, d_model]. Inclusive - ie, start_at_layer = 0 skips the embedding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t"><span class="str">                then runs the rest of the model. Supports negative indexing. start_at_layer = -1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t"><span class="str">                only runs the final block and the unembedding. Defaults to None (run the full</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t"><span class="str">                model).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t"><span class="str">            tokens: Optional[Int[torch.Tensor, "batch pos"]]: Tokenized input. Only use if</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t"><span class="str">                start_at_layer is not None and return type is "loss" or "both".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t"><span class="str">            shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]]: Positional</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t"><span class="str">                embedding for shortformer models. Only use if start_at_layer is not None and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t"><span class="str">                self.cfg.positional_embedding_type == "shortformer".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t"><span class="str">            attention_mask: Optional[torch.Tensor]: Override the attention mask used to ignore</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t"><span class="str">                padded tokens. If start_at_layer is not None and (self.tokenizer.padding_side ==</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t"><span class="str">                "left" or past_kv_cache is not None), this should be passed as the attention mask</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t"><span class="str">                is not computed automatically. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t"><span class="str">            stop_at_layer Optional[int]: If not None, stop the forward pass at the specified layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t"><span class="str">                Exclusive - ie, stop_at_layer = 0 will only run the embedding layer, stop_at_layer =</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t"><span class="str">                1 will run the embedding layer and the first transformer block, etc. Supports</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t"><span class="str">                negative indexing. Useful for analysis of intermediate layers, eg finding neuron</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t"><span class="str">                activations in layer 3 of a 24 layer model. Defaults to None (run the full model).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t"><span class="str">                If not None, we return the last residual stream computed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t"><span class="str">            past_kv_cache Optional[HookedTransformerKeyValueCache]: If not None, keys and values</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t"><span class="str">                will be stored for every attention head (unless the cache is frozen). If there are</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t"><span class="str">                keys and values already in the cache, these will be prepended to the keys and values</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t"><span class="str">                for the new input, so that the new tokens can pay attention to previous tokens. This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t"><span class="str">                is useful for generating text, because we don't need to repeat computation for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t"><span class="str">                tokens that have already been through the model. Also caches attention_mask so</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t"><span class="str">                previous tokens are masked correctly (unless frozen). Padding should be ignored in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t"><span class="str">                all cases, so it's okay to eg. pass in left padded tokens twice in a row.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t"><span class="str">                Warning: Don't accidentally prepend_bos to the second half of a prompt.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t"><span class="str">                Defaults to None (don't use caching).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">            <span class="key">if</span> <span class="nam">start_at_layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">                <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">                    <span class="nam">residual</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">                <span class="op">)</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">input_to_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">                    <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">                    <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">                    <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">=</span><span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">                    <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">                <span class="key">assert</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">            <span class="key">if</span> <span class="nam">start_at_layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">                <span class="nam">start_at_layer</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">            <span class="com"># If we explicitly want to start or stop at a layer, we only iterate through the blocks</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="com"># between those indices. Note that start_at_layer is inclusive and stop_at_layer is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">            <span class="com"># exclusive.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="com"># Eg: start_at_layer==None + stop_at_layer==0 means to only run the embed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="com"># Eg: start_at_layer==3 + stop_at_layer==-1 means to run from layer 3 until the end of the PENULTIMATE layer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">            <span class="nam">blocks_and_idxs</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">zip</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">            <span class="key">for</span> <span class="nam">i</span><span class="op">,</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">blocks_and_idxs</span><span class="op">[</span><span class="nam">start_at_layer</span><span class="op">:</span><span class="nam">stop_at_layer</span><span class="op">]</span><span class="op">:</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">                <span class="com"># Note that each block includes skip connections, so we don't need</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">                <span class="com"># residual + block(residual)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">                <span class="com"># If we're using multiple GPUs, we need to send the residual and shortformer_pos_embed to the correct GPU</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">residual</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">                <span class="key">if</span> <span class="nam">shortformer_pos_embed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="nam">shortformer_pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">block</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">                    <span class="nam">residual</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">                    <span class="com"># Cache contains a list of HookedTransformerKeyValueCache objects, one for each</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">                    <span class="com"># block</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">                    <span class="nam">past_kv_cache_entry</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span> <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span><span class="op">=</span><span class="nam">shortformer_pos_embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">=</span><span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">                <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_layer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">                <span class="com"># When we stop at an early layer, we end here rather than doing further computation</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">                <span class="key">return</span> <span class="nam">residual</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span><span class="op">(</span><span class="nam">residual</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">                <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">                <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">(</span><span class="nam">residual</span><span class="op">)</span>  <span class="com"># [batch, pos, d_vocab]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">595&#x202F;&#x219B;&#x202F;596</span><span class="annotate long">line 595 didn't jump to line 596, because the condition on line 595 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">*</span> <span class="nam">F</span><span class="op">.</span><span class="nam">tanh</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">                <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"logits"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">                    <span class="key">return</span> <span class="nam">logits</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">                        <span class="nam">tokens</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"tokens must be passed in if return_type is 'loss' or 'both'"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">                    <span class="nam">loss</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">loss_fn</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">,</span> <span class="nam">per_token</span><span class="op">=</span><span class="nam">loss_per_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">                    <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"loss"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">606&#x202F;&#x219B;&#x202F;608</span><span class="annotate long">line 606 didn't jump to line 608, because the condition on line 606 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">                        <span class="key">return</span> <span class="nam">loss</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">                    <span class="key">elif</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"both"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">                        <span class="key">return</span> <span class="nam">Output</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">loss</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">                        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">f"Invalid return_type passed in: {return_type}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">                        <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">    <span class="key">def</span> <span class="nam">loss_fn</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="nam">logits</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">        <span class="nam">per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">        <span class="str">"""Wrapper around `utils.lm_cross_entropy_loss`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t"><span class="str">        Used in forward() with return_type=="loss" or "both".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span> <span class="op">!=</span> <span class="nam">logits</span><span class="op">.</span><span class="nam">device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">625&#x202F;&#x219B;&#x202F;626</span><span class="annotate long">line 625 didn't jump to line 626, because the condition on line 625 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">logits</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">        <span class="key">return</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">lm_cross_entropy_loss</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">,</span> <span class="nam">per_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">True</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Output</span><span class="op">,</span> <span class="nam">ActivationCache</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">False</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Output</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">        <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">            <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">            <span class="nam">Loss</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">            <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">        <span class="nam">Union</span><span class="op">[</span><span class="nam">ActivationCache</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">        <span class="str">"""Wrapper around `run_with_cache` in HookedRootModule.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t"><span class="str">        If return_cache_object is True, this will return an ActivationCache object, with a bunch of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t"><span class="str">        useful HookedTransformer specific methods, otherwise it will return a dictionary of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t"><span class="str">        activations as in HookedRootModule.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">        <span class="nam">out</span><span class="op">,</span> <span class="nam">cache_dict</span> <span class="op">=</span> <span class="nam">super</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">            <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_cache_object</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">661&#x202F;&#x219B;&#x202F;665</span><span class="annotate long">line 661 didn't jump to line 665, because the condition on line 661 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="nam">ActivationCache</span><span class="op">(</span><span class="nam">cache_dict</span><span class="op">,</span> <span class="nam">self</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">=</span><span class="key">not</span> <span class="nam">remove_batch_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">            <span class="key">return</span> <span class="nam">out</span><span class="op">,</span> <span class="nam">cache</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">            <span class="key">return</span> <span class="nam">out</span><span class="op">,</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">        <span class="str">"""Set the tokenizer to use for this model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t"><span class="str">            tokenizer (PreTrainedTokenizer): a pretrained HuggingFace tokenizer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t"><span class="str">            default_padding_side (str): "right" or "left", which side to pad on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">PreTrainedTokenizerBase</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"{type(tokenizer)} is not a supported tokenizer, please use PreTrainedTokenizer or PreTrainedTokenizerFast"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="key">assert</span> <span class="nam">default_padding_side</span> <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">            <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">            <span class="str">"left"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="op">]</span><span class="op">,</span> <span class="str">f"padding_side must be 'right' or 'left', got {default_padding_side}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">        <span class="com"># Use a tokenizer that is initialized with add_bos_token=True as the default tokenizer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">        <span class="com"># Such a tokenizer should be set as the default tokenizer because the tokenization of some</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">        <span class="com"># tokenizers like LlamaTokenizer are different when bos token is automatically/manually</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">        <span class="com"># prepended, and add_bos_token cannot be dynamically controlled after initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">        <span class="com"># (https://github.com/huggingface/transformers/issues/25886).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="nam">tokenizer_with_bos</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_tokenizer_with_bos</span><span class="op">(</span><span class="nam">tokenizer</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="op">=</span> <span class="nam">tokenizer_with_bos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">padding_side</span> <span class="op">=</span> <span class="nam">default_padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t">        <span class="com"># Some tokenizers doesn't automatically prepend the BOS token even when they are initialized</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">        <span class="com"># with add_bos_token=True. Therefore, we need this information to dynamically control prepend_bos.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">encode</span><span class="op">(</span><span class="str">""</span><span class="op">)</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">702&#x202F;&#x219B;&#x202F;703</span><span class="annotate long">line 702 didn't jump to line 703, because the condition on line 702 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span> <span class="op">=</span> <span class="str">"&lt;|endoftext|>"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">706&#x202F;&#x219B;&#x202F;707</span><span class="annotate long">line 706 didn't jump to line 707, because the condition on line 706 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="com"># Infer vocab size from tokenizer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">=</span> <span class="nam">max</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">vocab</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">+</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab_out</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">        <span class="nam">truncate</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">        <span class="str">"""Converts a string to a tensor of tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t"><span class="str">        If prepend_bos is True, prepends the BOS token to the input - this is recommended when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t"><span class="str">        creating a sequence of tokens to be input to a model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t"><span class="str">        Gotcha: prepend_bos prepends a beginning of string token. This is a recommended default when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t"><span class="str">        inputting a prompt to the model as the first token is often treated weirdly, but should only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t"><span class="str">        be done at the START of the prompt. Make sure to turn it off if you're looking at the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t"><span class="str">        tokenization of part of the prompt! (Note: some models eg GPT-2 were not trained with a BOS</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t"><span class="str">        token, others (OPT and my models) were)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t"><span class="str">        Gotcha2: Tokenization of a string depends on whether there is a preceding space and whether</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t"><span class="str">        the first letter is capitalized. It's easy to shoot yourself in the foot here if you're not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t"><span class="str">        careful!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t"><span class="str">            input (Union[str, List[str]]): The input to tokenize.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t"><span class="str">                multiple strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t"><span class="str">            move_to_device (bool): Whether to move the output tensor of tokens to the device the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t"><span class="str">                model lives on. Defaults to True truncate (bool): If the output tokens are too long,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t"><span class="str">                whether to truncate the output tokens to the model's max context window. Does nothing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t"><span class="str">                for shorter inputs. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">,</span> <span class="str">"Cannot use to_tokens without a tokenizer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Set the tokenizer for the model by calling set_tokenizer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">                <span class="com"># We want to prepend bos but the tokenizer doesn't automatically do it, so we add it manually</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">                <span class="nam">input</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_input_with_manually_prepended_bos</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">input</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">                <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">                <span class="nam">return_tensors</span><span class="op">=</span><span class="str">"pt"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">                <span class="nam">padding</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">                <span class="nam">truncation</span><span class="op">=</span><span class="nam">truncate</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">                <span class="nam">max_length</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_ctx</span> <span class="key">if</span> <span class="nam">truncate</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="str">"input_ids"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">                <span class="com"># We don't want to prepend bos but the tokenizer does it automatically, so we remove it manually</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_tokens_with_bos_removed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">            <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">            <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_string</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">            <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">        <span class="str">"""Tokens to String(s).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t"><span class="str">        Converts a tensor of tokens to a string (if rank 1) or a list of strings (if rank 2).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t"><span class="str">        Accepts lists of tokens and numpy arrays as inputs too (and converts to tensors internally)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">,</span> <span class="str">"Cannot use to_string without a tokenizer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">            <span class="com"># We allow lists to be input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="com"># I'm not sure what exactly clean_up_tokenization_spaces does, but if</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">        <span class="com"># it's set, then tokenization is no longer invertible, and some tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">        <span class="com"># with a bunch of whitespace get collapsed together</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">2</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">batch_decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">        <span class="key">elif</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">&lt;=</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">808&#x202F;&#x219B;&#x202F;811</span><span class="annotate long">line 808 didn't jump to line 811, because the condition on line 808 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid shape passed in: {tokens.shape}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_str_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">            <span class="nam">list</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">        <span class="str">"""Map text, a list of text or tokens to a list of tokens as strings.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t"><span class="str">        Gotcha: prepend_bos prepends a beginning of string token. This is a recommended default when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t"><span class="str">        inputting a prompt to the model as the first token is often treated weirdly, but should only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t"><span class="str">        be done at the START of the prompt. If prepend_bos=None is passed, it implies the usage of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t"><span class="str">        self.cfg.default_prepend_bos which is set to True unless specified otherwise. Therefore,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t"><span class="str">        make sure to locally turn it off by passing prepend_bos=False if you're looking at the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t"><span class="str">        tokenization of part of the prompt! (Note: some models eg GPT-2 were not trained with a BOS</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t"><span class="str">        token, others (OPT and my models) were)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t"><span class="str">        Gotcha2: Tokenization of a string depends on whether there is a preceding space and whether</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t"><span class="str">        the first letter is capitalized. It's easy to shoot yourself in the foot here if you're not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t"><span class="str">        careful!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t"><span class="str">        Gotcha3: If passing a string that exceeds the model's context length (model.cfg.n_ctx), it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t"><span class="str">        will be truncated.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t"><span class="str">            input (Union[str, list, torch.Tensor]): The input - either a string or a tensor of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t"><span class="str">                tokens. If tokens, should be a tensor of shape [pos] or [1, pos].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t"><span class="str">            str_tokens: List of individual tokens as strings</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">            <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">                <span class="key">return</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">                    <span class="nam">map</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">                        <span class="key">lambda</span> <span class="nam">tokens</span><span class="op">:</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_str_tokens</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">                        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">                <span class="op">)</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">                    <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">                <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">                <span class="com"># Gemma tokenizer expects a batch dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">                <span class="key">if</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">name_or_path</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">874&#x202F;&#x219B;&#x202F;875</span><span class="annotate long">line 874 didn't jump to line 875, because the condition on line 874 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>  <span class="com"># Get rid of a trivial batch dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">                <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">                    <span class="com"># Don't pass dimensionless tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">885&#x202F;&#x219B;&#x202F;895</span><span class="annotate long">line 885 didn't jump to line 895, because the condition on line 885 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>  <span class="com"># Get rid of a trivial batch dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">                <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">                    <span class="com"># Don't pass dimensionless tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">expand_dims</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">axis</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid input type to to_str_tokens: {type(input)}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">            <span class="nam">str_tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">batch_decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">            <span class="key">return</span> <span class="nam">str_tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_single_token</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">string</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">        <span class="str">"""Map a string that makes up a single token to the id for that token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t"><span class="str">        Raises an error for strings that are not a single token! If uncertain use to_tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">        <span class="com"># We use the to_tokens method, do not append a BOS token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">        <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">string</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">        <span class="com"># If token shape is non-empty, raise error</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">token</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span> <span class="str">f"Input string: {string} is not a single token!"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">        <span class="key">return</span> <span class="nam">token</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_single_str_token</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">int_token</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">        <span class="com"># Gives the single token corresponding to an int in string form</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">int_token</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">        <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_str_tokens</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="op">[</span><span class="nam">int_token</span><span class="op">]</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">        <span class="key">assert</span> <span class="nam">len</span><span class="op">(</span><span class="nam">token</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">        <span class="key">return</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">str</span><span class="op">,</span> <span class="nam">token</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_token_position</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">        <span class="nam">single_token</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">int</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">        <span class="nam">mode</span><span class="op">=</span><span class="str">"first"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">        <span class="str">"""Get the position of a single_token in a string or sequence of tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t"><span class="str">        Raises an error if the token is not present.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t"><span class="str">        Gotcha: If you're inputting a string, it'll automatically be tokenized. Be careful about the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t"><span class="str">        setting for prepend_bos! When a string is input to the model, a BOS (beginning of sequence)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t"><span class="str">        token is prepended by default when the string is tokenized because</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t"><span class="str">        self.cfg.default_prepend_bos is set to True unless specified otherwise. But this should only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t"><span class="str">        be done at the START of the input, not when inputting part of the prompt. If you're getting</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t"><span class="str">        weird off-by-one errors, check carefully for what the setting should be!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t"><span class="str">            single_token (Union[str, int]): The token to search for. Can</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t"><span class="str">                be a token index, or a string (but the string must correspond to a single token).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t"><span class="str">            input (Union[str, torch.Tensor]): The sequence to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t"><span class="str">                search in. Can be a string or a rank 1 tensor of tokens or a rank 2 tensor of tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t"><span class="str">                with a dummy batch dimension.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t"><span class="str">            mode (str, optional): If there are multiple matches, which match to return. Supports</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t"><span class="str">                "first" or "last". Defaults to "first".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="com"># If the input is a string, convert to tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">2</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="com"># If the tokens have shape [1, seq_len], flatten to [seq_len]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">                <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"If tokens are rank two, they must have shape [1, seq_len], not {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">single_token</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="com"># If the single token is a string, convert to an integer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">            <span class="nam">single_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">single_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">single_token</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">969&#x202F;&#x219B;&#x202F;970</span><span class="annotate long">line 969 didn't jump to line 970, because the condition on line 969 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">            <span class="nam">single_token</span> <span class="op">=</span> <span class="nam">single_token</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">        <span class="nam">indices</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="nam">tokens</span> <span class="op">==</span> <span class="nam">single_token</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">        <span class="key">assert</span> <span class="nam">len</span><span class="op">(</span><span class="nam">indices</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span><span class="op">,</span> <span class="str">"The token does not occur in the prompt"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"first"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">            <span class="key">return</span> <span class="nam">indices</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"last"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">976&#x202F;&#x219B;&#x202F;979</span><span class="annotate long">line 976 didn't jump to line 979, because the condition on line 976 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="key">return</span> <span class="nam">indices</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be 'first' or 'last', not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">    <span class="key">def</span> <span class="nam">tokens_to_residual_directions</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">        <span class="str">"""Map tokens to a tensor with the unembedding vector for those tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t"><span class="str">        I.e. the vector in the residual stream that we dot with to the get the logit for that token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t"><span class="str">        WARNING: If you use this without folding in LayerNorm, the results will be misleading and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t"><span class="str">        may be incorrect, as the LN weights change the unembed map. This is done automatically with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t"><span class="str">        the fold_ln flag on from_pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t"><span class="str">        WARNING 2: LayerNorm scaling will scale up or down the effective direction in the residual</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t"><span class="str">        stream for each output token on any given input token position.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t"><span class="str">        ActivationCache.apply_ln_to_stack will apply the appropriate scaling to these directions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t"><span class="str">            tokens (Union[str, int, torch.Tensor]): The token(s). If a single token, can be a single</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t"><span class="str">                element tensor, an integer, or string. If string, will be mapped to a single token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t"><span class="str">                using to_single_token, and an error raised if it's multiple tokens. The method also</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t"><span class="str">                works for a batch of input tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t"><span class="str">            residual_direction torch.Tensor: The unembedding vector for the token(s), a stack of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t"><span class="str">                [d_model] tensor.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">numel</span><span class="op">(</span><span class="op">)</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="com"># If the tokens are a tensor, and have more than one element, assume they are a batch of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="com"># tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="nam">residual_directions</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_U</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">            <span class="nam">residual_directions</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">                <span class="nam">residual_directions</span><span class="op">,</span> <span class="str">"d_model ... -> ... d_model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_directions</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="com"># Otherwise there is a single token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1027&#x202F;&#x219B;&#x202F;1028</span><span class="annotate long">line 1027 didn't jump to line 1028, because the condition on line 1027 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1029&#x202F;&#x219B;&#x202F;1030</span><span class="annotate long">line 1029 didn't jump to line 1030, because the condition on line 1029 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">numel</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1031&#x202F;&#x219B;&#x202F;1034</span><span class="annotate long">line 1031 didn't jump to line 1034, because the condition on line 1031 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid token type: {type(tokens)}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">            <span class="nam">residual_direction</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_U</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="nam">token</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_direction</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">    <span class="key">def</span> <span class="nam">to</span><span class="op">(</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">        <span class="nam">device_or_dtype</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">        <span class="nam">print_details</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">        <span class="key">return</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">move_to_and_update_config</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">device_or_dtype</span><span class="op">,</span> <span class="nam">print_details</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">    <span class="key">def</span> <span class="nam">cuda</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">        <span class="str">"""Wrapper around cuda that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"cuda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">    <span class="key">def</span> <span class="nam">cpu</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">        <span class="str">"""Wrapper around cuda that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"cpu"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">    <span class="key">def</span> <span class="nam">mps</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">        <span class="str">"""Wrapper around mps that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"mps"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">    <span class="key">def</span> <span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">        <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="str">"ln_final"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span><span class="op">,</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">enumerate</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">            <span class="nam">block</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">        <span class="nam">cls</span><span class="op">:</span> <span class="nam">Type</span><span class="op">[</span><span class="nam">T</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">        <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">        <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">        <span class="nam">hf_model</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">AutoModelForCausalLM</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">        <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">PreTrainedTokenizerBase</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="str">"float32"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">        <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">T</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">        <span class="str">"""Load in a Pretrained Model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t"><span class="str">        Load in pretrained model weights to the HookedTransformer format and optionally to do some</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t"><span class="str">        processing to make the model easier to interpret. Currently supports loading from most</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t"><span class="str">        autoregressive HuggingFace models (``gpt2``, ``neo``, ``gptj``, ``opt``...) and from a range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t"><span class="str">        of toy models and SoLU models trained by Neel Nanda. The full list is available in the docs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t"><span class="str">        under :doc:`model properties&lt;/generated/model_properties_table>`. Also supports loading from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t"><span class="str">        a checkpoint for checkpointed models (currently, models trained by NeelNanda and the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t"><span class="str">        stanford-crfm models (using parameters ``checkpoint_index`` and ``checkpoint_value``).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t"><span class="str">        See :meth:`load_and_process_state_dict` for details on the processing (folding layer norm,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t"><span class="str">        centering the unembedding and centering the writing weights).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t"><span class="str">            model_name: The model name - must be an element of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t"><span class="str">                :const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` or an alias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t"><span class="str">                of one. The full list of available models can be found in the docs under :doc:`model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t"><span class="str">                properties&lt;/generated/model_properties_table>`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t"><span class="str">            fold_ln: Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t"><span class="str">                `LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t"><span class="str">                &lt;https://wandb.ai/wandb_fc/LayerNorm/reports/Layer-Normalization-in-Pytorch-With-Examples---VmlldzoxMjk5MTk1>`_</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t"><span class="str">                is a common regularization technique used in transformers. Unlike BatchNorm, it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t"><span class="str">                cannot be turned off at inference time, as it significantly alters the mathematical</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t"><span class="str">                function implemented by the transformer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t"><span class="str">                When `fold_ln` is set to True, LayerNorm (with weights :math:`w_{ln}` and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t"><span class="str">                :math:`b_{ln}`) followed by a linear layer (:math:`W + b`) is optimized to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t"><span class="str">                LayerNormPre (just centering &amp; normalizing) followed by a new linear layer with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t"><span class="str">                :math:`W_{eff} = w[:, \text{None}] * W` (element-wise multiplication) and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t"><span class="str">                :math:`b_{eff} = b + b_{ln} @ W`. This transformation is computationally equivalent</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t"><span class="str">                and simplifies the model's interpretability. It essentially merges LayerNorm weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t"><span class="str">                into the subsequent linear layer's weights, which is handled by HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t"><span class="str">                when loading pre-trained weights. Set `fold_ln` to False when loading a state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t"><span class="str">                if you wish to turn this off.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t"><span class="str">                Mathematically, LayerNorm is defined as follows:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t"><span class="str">                .. math::</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t"><span class="str">                    x_1 &amp;= x_0 - \\text{mean}(x_0)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t"><span class="str">                    x_2 &amp;= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t"><span class="str">                    x_3 &amp;= x_2 \\cdot w</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t"><span class="str">                    x_4 &amp;= x_3 + b</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t"><span class="str">                For further details, refer to `this document</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t"><span class="str">                &lt;https://transformer-circuits.pub/2021/framework/index.html#:~:text=Handling%20Layer%20Normalization>`_.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t"><span class="str">            center_writing_weights: Whether to center weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t"><span class="str">                writing to the residual stream (ie set mean to be zero). Due to LayerNorm this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t"><span class="str">                doesn't change the computation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t"><span class="str">                A related idea to folding layernorm (``fold_ln``) - *every* component reading an</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t"><span class="str">                input from the residual stream is preceded by a LayerNorm, which means that the mean</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t"><span class="str">                of a residual stream vector (ie the component in the direction of all ones) never</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t"><span class="str">                matters. This means we can remove the all ones component of weights and biases whose</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t"><span class="str">                output *writes* to the residual stream. Mathematically, ``W_writing -=</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t"><span class="str">                W_writing.mean(dim=1, keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t"><span class="str">            center_unembed: Whether to center W_U (ie set mean</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t"><span class="str">                to be zero). Softmax is translation invariant so this doesn't affect log probs or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t"><span class="str">                loss, but does change logits.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t"><span class="str">                The logits are fed into a softmax. Softmax is translation invariant (eg, adding 1 to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t"><span class="str">                every logit doesn't change the output), so we can simplify things by setting the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t"><span class="str">                mean of the logits to be zero. This is equivalent to setting the mean of every</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t"><span class="str">                output vector of ``W_U`` to zero. In code, ``W_U -= W_U.mean(dim=-1,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t"><span class="str">                keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices: Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t"><span class="str">            checkpoint_index: If loading from a checkpoint, the index of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t"><span class="str">                the checkpoint to load.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t"><span class="str">            checkpoint_value: If loading from a checkpoint, the value of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t"><span class="str">                the checkpoint to load, ie the step or token number (each model has checkpoints</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t"><span class="str">                labelled with exactly one of these). E.g. ``1000`` for a checkpoint taken at step</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t"><span class="str">                1000 or after 1000 tokens. If `checkpoint_index` is also specified, this will be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t"><span class="str">                ignored.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t"><span class="str">            hf_model: If you have already loaded in the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t"><span class="str">                HuggingFace model, you can pass it in here rather than needing to recreate the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t"><span class="str">                object. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t"><span class="str">            device: The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t"><span class="str">                default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t"><span class="str">            n_devices: The number of devices to split the model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t"><span class="str">                across. Defaults to 1. If greater than 1, `device` must be cuda.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t"><span class="str">                provided, it is inferred from cfg.tokenizer_name or initialized to None. If None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t"><span class="str">                then the model cannot be passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t"><span class="str">                cfg. device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t"><span class="str">            fold_value_biases: Each attention head has a value bias. Values are averaged to create</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t"><span class="str">                mixed values (``z``), weighted by the attention pattern, but as the bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t"><span class="str">                constant, its contribution to ``z`` is exactly the same. The output of a head is ``z</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t"><span class="str">                @ W_O``, and so the value bias just linearly adds to the output of the head. This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t"><span class="str">                means that the value bias of a head has nothing to do with the head, and is just a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t"><span class="str">                constant added to the attention layer outputs. We can take the sum across these and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t"><span class="str">                b_O to get an "effective bias" for the layer. In code, we set ``b_V=0``. and ``b_O =</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t"><span class="str">                (b_V @ W_O).sum(dim=0) + b_O``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t"><span class="str">                The technical derivation of this is as follows. ``v = residual @ W_V[h] +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t"><span class="str">                broadcast_b_V[h]`` for each head ``h`` (where ``b_V`` is broadcast up from shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t"><span class="str">                ``d_head`` to shape ``[position, d_head]``). And ``z = pattern[h] @ v = pattern[h] @</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t"><span class="str">                residual @ W_V[h] + pattern[h] @ broadcast_b_V[h]``. Because ``pattern[h]`` is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t"><span class="str">                ``[destination_position, source_position]`` and ``broadcast_b_V`` is constant along</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t"><span class="str">                the ``(source_)position`` dimension, we're basically just multiplying it by the sum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t"><span class="str">                of the pattern across the ``source_position`` dimension, which is just ``1``. So it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t"><span class="str">                remains exactly the same, and so is just broadcast across the destination positions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t"><span class="str">            default_prepend_bos: Default behavior of whether to prepend the BOS</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t"><span class="str">                token when the methods of HookedTransformer process input text to tokenize (only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t"><span class="str">                when input is a string).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t"><span class="str">                Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t"><span class="str">                1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t"><span class="str">                2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t"><span class="str">                3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t"><span class="str">                Even for models not explicitly trained with the BOS token, heads often use the first position as a resting position</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t"><span class="str">                and accordingly lose information from the first token, so this empirically seems to give better</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t"><span class="str">                results. Note that you can also locally override the default behavior by passing in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t"><span class="str">                prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t"><span class="str">            from_pretrained_kwargs: Any other optional argument passed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t"><span class="str">                HuggingFace's from_pretrained (e.g. "cache_dir" or "torch_dtype"). Also passed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t"><span class="str">                other HuggingFace functions when compatible. For some models or arguments it doesn't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t"><span class="str">                work, especially for models that are not internally loaded with HuggingFace's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t"><span class="str">                from_pretrained (e.g. SoLU models).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t"><span class="str">            dtype: What data type to load the model in (also sets the dtype of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t"><span class="str">                the HuggingFace model). Set to bfloat16 or float16 if you get out of memory errors when loading</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t"><span class="str">                the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on when tokenizing. Defaults to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t"><span class="str">                "right".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t"><span class="str">            first_n_layers: If specified, only load the first n layers of the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">        <span class="key">if</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"t5"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1230&#x202F;&#x219B;&#x202F;1231</span><span class="annotate long">line 1230 didn't jump to line 1231, because the condition on line 1230 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">            <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">                <span class="str">"Execution stopped: Please use HookedEncoderDecoder to load T5 models instead of HookedTransformer."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="key">or</span> <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization not supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">        <span class="key">if</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1240&#x202F;&#x219B;&#x202F;1241</span><span class="annotate long">line 1240 didn't jump to line 1241, because the condition on line 1240 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">config</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="nam">qc</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="nam">load_in_4bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">            <span class="nam">load_in_8bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">            <span class="nam">quant_method</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="nam">load_in_8bit</span><span class="op">,</span> <span class="str">"8-bit quantization is not supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">__version__</span><span class="op">)</span> <span class="op">&lt;</span> <span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="str">"2.1.1"</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for torch versions >= 2.1.1"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="str">"llama"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for Llama models"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">            <span class="key">if</span> <span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">                    <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span> <span class="op">==</span> <span class="str">"bitsandbytes"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Only bitsandbytes quantization is supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">dtype</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="com"># Convert from string to a torch dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">DTYPE_FROM_STRING</span><span class="op">[</span><span class="nam">dtype</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">        <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">from_pretrained_kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1263&#x202F;&#x219B;&#x202F;1266</span><span class="annotate long">line 1263 didn't jump to line 1266, because the condition on line 1263 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="com"># For backwards compatibility with the previous way to do low precision loading</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="com"># This should maybe check the user did not explicitly set dtype *and* torch_dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">from_pretrained_kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1268&#x202F;&#x219B;&#x202F;1272</span><span class="annotate long">line 1268 didn't jump to line 1272, because the condition on line 1268 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">            <span class="op">(</span><span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"torch_dtype"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">            <span class="key">or</span> <span class="nam">dtype</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">        <span class="op">)</span> <span class="key">and</span> <span class="nam">device</span> <span class="key">in</span> <span class="op">[</span><span class="str">"cpu"</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"float16 models may not work on CPU. Consider using a GPU or bfloat16."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">        <span class="com"># Get the model name used in HuggingFace, rather than the alias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">        <span class="com"># Load the config into an HookedTransformerConfig object. If loading from a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">        <span class="com"># checkpoint, the config object will contain the information about the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">        <span class="com"># checkpoint</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">        <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="nam">hf_cfg</span><span class="op">=</span><span class="nam">hf_cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="nam">checkpoint_index</span><span class="op">=</span><span class="nam">checkpoint_index</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="nam">checkpoint_value</span><span class="op">=</span><span class="nam">checkpoint_value</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="nam">n_devices</span><span class="op">=</span><span class="nam">n_devices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="nam">first_n_layers</span><span class="op">=</span><span class="nam">first_n_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">                    <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">                    <span class="str">"ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">                <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">                    <span class="str">"You tried to specify center_unembed=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">                    <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">                <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">                    <span class="str">"You tried to specify center_writing_weights=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">                    <span class="str">"Setting center_writing_weights=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">                <span class="nam">center_writing_weights</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span> <span class="key">and</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1313&#x202F;&#x219B;&#x202F;1314</span><span class="annotate long">line 1313 didn't jump to line 1314, because the condition on line 1313 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">                <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">            <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">        <span class="com"># Get the state dict of the model (ie a mapping of parameter names to tensors), processed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">        <span class="com"># match the HookedTransformer parameter names.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">,</span> <span class="nam">hf_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span> <span class="op">**</span><span class="nam">from_pretrained_kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">        <span class="com"># Create the HookedTransformer object</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">        <span class="nam">model</span> <span class="op">=</span> <span class="nam">cls</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="nam">cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="nam">move_to_device</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">        <span class="nam">model</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1343&#x202F;&#x219B;&#x202F;1346</span><span class="annotate long">line 1343 didn't jump to line 1346, because the condition on line 1343 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">            <span class="nam">model</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">        <span class="nam">print</span><span class="op">(</span><span class="str">f"Loaded pretrained model {model_name} into HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">        <span class="key">return</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained_no_processing</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">        <span class="nam">cls</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">        <span class="str">"""Wrapper for from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t"><span class="str">        Wrapper for from_pretrained with all boolean flags related to simplifying the model set to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t"><span class="str">        False. Refer to from_pretrained for details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">        <span class="key">return</span> <span class="nam">cls</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">            <span class="nam">model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">    <span class="key">def</span> <span class="nam">init_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">        <span class="str">"""Initialize weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t"><span class="str">        LayerNorm weights are already initialized to 1.0, and all biases are initialized to 0.0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t"><span class="str">        (including LayerNorm), so this just initializes weight matrices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t"><span class="str">        Weight matrices are set to empty by default (to save space + compute, since they're the bulk</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t"><span class="str">        of the parameters), so it is important to call this if you are not loading in pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t"><span class="str">        weights! Note that this function assumes that weight names being with `W_`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t"><span class="str">        Set seed here to ensure determinism.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t"><span class="str">        This does NOT follow the PyTorch scheme, which as far as I can tell is super out of date but</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t"><span class="str">        no one has gotten round to updating it? https://github.com/pytorch/pytorch/issues/18182</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t"><span class="str">        The default PyTorch scheme is the following: all linear layers use uniform(-1/sqrt(fan_in),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t"><span class="str">        1/sqrt(fan_in)) for weights, and uniform(-1/sqrt(fan_in), 1/sqrt(fan_in)) for biases. For</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t"><span class="str">        biases, fan_in is computed using the fan_in for the weight matrix of the linear layer. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t"><span class="str">        tha it *does not actually* use Kaiming initialization, despite the fact that it calls the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t"><span class="str">        function.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t"><span class="str">        However, for Transformer blocks, it instead initializes biases to zero and weights using Xavier uniform, that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t"><span class="str">        is: uniform(-sqrt(6 / (fan_in + fan_out)), sqrt(6 / (fan_in + fan_out))) for weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t"><span class="str">        PyTorch Transformers are especially bad - TransformerEncoder initializes all layers to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t"><span class="str">        exact same weights?! https://github.com/pytorch/pytorch/issues/72253.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t"><span class="str">        The best paper I've found on transformer initialization is the muP paper, but haven't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t"><span class="str">        integrated those ideas yet: https://arxiv.org/abs/2203.03466</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t"><span class="str">        We split off the initialization into separate functions because muP initialization handles</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t"><span class="str">        different parts of the model differently.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1416&#x202F;&#x219B;&#x202F;1417</span><span class="annotate long">line 1416 didn't jump to line 1417, because the condition on line 1416 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">manual_seed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1419&#x202F;&#x219B;&#x202F;1421</span><span class="annotate long">line 1419 didn't jump to line 1421, because the condition on line 1419 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"muP"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>  <span class="com"># muP uses normal initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">        <span class="str">"""Initialize weights with GPT-2 initialization. Biases are initialized to 0.0 and weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t"><span class="str">        are initialized to N(0, 0.64/d_model) if initializer_range is not set, otherwise std is initializer_range.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">                <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t"><span class="str">        Initialize weights with Xavier initialization -- that is, scale the weights by sqrt(6 /</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t"><span class="str">        (fan_in + fan_out)) for a [-1, 1] uniform distribution, or sqrt(2 / (fan_in + fan_out)) for a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t"><span class="str">        standard normal.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t"><span class="str">        Note that since TransformerLens implements the matrices in the opposite orientation to what</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t"><span class="str">        torch does (e.g. it's d_in x d_out, not d_out x d_in as in torch), we need to calculate it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t"><span class="str">        ourselves.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">                    <span class="nam">init_xavier_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">                    <span class="nam">init_xavier_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t"><span class="str">        Initialize weights with Kaiming initialization -- that is, scale the weights by</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t"><span class="str">        c / sqrt(fan_in), where c = sqrt(2) if the params were immediately preceded by a relu and 1 for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t"><span class="str">        everything else.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t"><span class="str">        Note that the numbers are actually incorrect here when you're using a nonlinearity other</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t"><span class="str">        than relu, e.g. the correct c for SiLu is ~1.74, for tanh it's 5/3 ~= 1.67, and for GeLU it's ~1.57.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t"><span class="str">        But this is unlikely to matter in practice.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t"><span class="str">        I'm just using fan_mode = "fan_in" for now, but it should be trivial to add fan_out.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t"><span class="str">        Again, we have to implement it ourselves because of the orientation of the matrices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">                    <span class="nam">init_kaiming_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">                    <span class="nam">init_kaiming_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t"><span class="str">        Initialize weights with muParameterization. This involves scaling output weights by a factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t"><span class="str">        of 1/fan_in, input weights and biases by 1, everything else by a factor of 1/sqrt(fan_in).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t"><span class="str">        Also, you need to use muAdamW, which rescales the learning rate for output weights and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t"><span class="str">        hidden weights by a factor of 1/fan_in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t"><span class="str">        All biases are still assumed to be initialized to 0.0, so we only need to change the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t"><span class="str">        weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">                <span class="nam">fan_in</span><span class="op">,</span> <span class="nam">_</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">calc_fan_in_and_fan_out</span><span class="op">(</span><span class="nam">param</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">                <span class="key">if</span> <span class="str">"embed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="nam">float</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">                <span class="key">elif</span> <span class="str">"unembed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">*=</span> <span class="num">3</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="op">-</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">        <span class="str">"""Load &amp; Process State Dict.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t"><span class="str">        Load a state dict into the model, and to apply processing to simplify it. The state dict is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t"><span class="str">        assumed to be in the HookedTransformer format.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="str">        See the relevant method (same name as the flag) for more details on the folding, centering</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t"><span class="str">        and processing flags.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t"><span class="str">            state_dict (dict): The state dict of the model, in HookedTransformer format. fold_ln</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="str">            fold_ln (bool, optional): Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="str">            center_writing_weights (bool, optional): Whether to center weights writing to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="str">                residual stream (ie set mean to be zero). Due to LayerNorm this doesn't change the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="str">                computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="str">            center_unembed (bool, optional): Whether to center W_U (ie set mean to be zero).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="str">                Softmax is translation invariant so this doesn't affect log probs or loss, but does</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="str">                change logits. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="str">            fold_value_biases (bool, optional): Whether to fold the value biases into the output</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="str">                bias. Because attention patterns add up to 1, the value biases always have a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t"><span class="str">                constant effect on a layer's output, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">                associated with. We can factor this all into a single output bias to the layer, and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="str">                make it easier to interpret the head's output.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices (bool, optional): Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t"><span class="str">            model_name (str, optional): checks the model name for special cases of state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t"><span class="str">                loading. Only used for Redwood 2L model currently.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span> <span class="key">and</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1544&#x202F;&#x219B;&#x202F;1545</span><span class="annotate long">line 1544 didn't jump to line 1545, because the condition on line 1544 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">                <span class="str">"With reduced precision, it is advised to use `from_pretrained_no_processing` instead of `from_pretrained`."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1549&#x202F;&#x219B;&#x202F;1554</span><span class="annotate long">line 1549 didn't jump to line 1554</span></span></p>
-    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">                <span class="str">"When running MoE models, it is advised to use a higher precision data type. See docs for more info."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1560&#x202F;&#x219B;&#x202F;1561</span><span class="annotate long">line 1560 didn't jump to line 1561, because the condition on line 1560 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">                    <span class="str">"You are using MoE, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1564&#x202F;&#x219B;&#x202F;1566</span><span class="annotate long">line 1564 didn't jump to line 1566, because the condition on line 1564 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">                    <span class="str">"You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1576&#x202F;&#x219B;&#x202F;1577</span><span class="annotate long">line 1576 didn't jump to line 1577, because the condition on line 1576 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">                    <span class="str">"You are not using LayerNorm, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">                    <span class="str">"This model is using final RMS normalization, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_unembed</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_value_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">        <span class="key">if</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1594&#x202F;&#x219B;&#x202F;1597</span><span class="annotate long">line 1594 didn't jump to line 1597, because the condition on line 1594 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">            <span class="com"># with quantization, parameters should be assigned</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">            <span class="com"># so that quantization settings are not lost</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">assign</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">            <span class="nam">state_dict_keys</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">state_dict_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">}</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">    <span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">        <span class="key">return</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">        <span class="str">"""Fold Layer Norm. Can also be used to fold RMS Norm, when fold_biases and center_weights are set to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t"><span class="str">        Takes in a state dict from a pretrained model, formatted to be consistent with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t"><span class="str">        HookedTransformer but with LayerNorm weights and biases. Folds these into the neighbouring</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t"><span class="str">        weights. See further_comments.md for more details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t"><span class="str">            state_dict (Dict[str, torch.Tensor]): State dict of pretrained model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t"><span class="str">            fold_biases (bool): Enables folding of LN biases. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t"><span class="str">            center_weights (bool): Enables the centering of weights after folding in LN. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="com"># Models that use Grouped Query Attention (Only Mistral at the time of writing) prefix their K/V weights and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">        <span class="com"># biases with an underscore in order to distinguish them, but folding the LN into them still works the same,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">        <span class="com"># so we just add the underscore if GQA is used (i.e. if `cfg.n_key_value_heads is specified`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">        <span class="nam">gqa</span> <span class="op">=</span> <span class="str">""</span> <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span> <span class="key">else</span> <span class="str">"_"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">            <span class="com"># Fold ln1 into attention - it's important to fold biases first, since biases depend on</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">            <span class="com"># weights but not vice versa The various indexing is just to broadcast ln.b and ln.w</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">            <span class="com"># along every axis other than d_model. Each weight matrix right multiplies. To fold in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">            <span class="com"># the bias, we use the W_ matrix to map it to the hidden space of the layer, so we need</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="com"># to sum along axis -2, which is the residual stream space axis.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1633&#x202F;&#x219B;&#x202F;1656</span><span class="annotate long">line 1633 didn't jump to line 1656</span></span></p>
-    <p class="run"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_K"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_V"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">            <span class="com"># Finally, we center the weights reading from the residual stream. The output of the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">            <span class="com"># first part of the LayerNorm is mean 0 and standard deviation 1, so the mean of any</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">            <span class="com"># input vector of the matrix doesn't matter and can be set to zero. Equivalently, the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">            <span class="com"># output of LayerNormPre is orthogonal to the vector of all 1s (because dotting with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">            <span class="com"># that gets the sum), so we can remove the component of the matrix parallel to this.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1674&#x202F;&#x219B;&#x202F;1692</span><span class="annotate long">line 1674 didn't jump to line 1692, because the condition on line 1674 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">            <span class="com"># Fold ln2 into MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1693&#x202F;&#x219B;&#x202F;1700</span><span class="annotate long">line 1693 didn't jump to line 1700</span></span></p>
-    <p class="run"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1704&#x202F;&#x219B;&#x202F;1705</span><span class="annotate long">line 1704 didn't jump to line 1705</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1712&#x202F;&#x219B;&#x202F;1720</span><span class="annotate long">line 1712 didn't jump to line 1720, because the condition on line 1712 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">                    <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">                        <span class="str">"d_model d_mlp -> 1 d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">                        <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"solu"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">                    <span class="com"># Fold ln3 into activation</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">                    <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1722&#x202F;&#x219B;&#x202F;1734</span><span class="annotate long">line 1722 didn't jump to line 1734</span></span></p>
-    <p class="run"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">                            <span class="str">f"blocks.{l}.mlp.b_out"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">                        <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">                            <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">                        <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">                            <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">                        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">                    <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1739&#x202F;&#x219B;&#x202F;1747</span><span class="annotate long">line 1739 didn't jump to line 1747, because the condition on line 1739 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">                        <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">                            <span class="str">"d_mlp d_model -> 1 d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">                            <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="com"># Fold ln_final into Unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span> <span class="key">and</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">            <span class="com"># Dumb bug from my old SoLU training code, some models have RMSNorm instead of LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">            <span class="com"># pre unembed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">            <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1761&#x202F;&#x219B;&#x202F;1767</span><span class="annotate long">line 1761 didn't jump to line 1767, because the condition on line 1761 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">            <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span><span class="op">,</span> <span class="str">"d_model d_vocab -> 1 d_vocab"</span><span class="op">,</span> <span class="str">"mean"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">        <span class="str">"""Center Writing Weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t"><span class="str">        Centers the weights of the model that write to the residual stream - W_out, W_E, W_pos and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t"><span class="str">        W_out. This is done by subtracting the mean of the weights from the weights themselves. This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t"><span class="str">        is done in-place. See fold_layer_norm for more details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">                <span class="str">"pos_embed.W_pos"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                <span class="str">f"blocks.{l}.attn.W_O"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">                <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">            <span class="op">)</span>  <span class="com"># W_O is [head_index, d_model, d_head]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">            <span class="op">)</span>  <span class="com"># b_O is [d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">                    <span class="str">f"blocks.{l}.mlp.W_out"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">                <span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">        <span class="str">"""Center the unembedding weights W_U.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t"><span class="str">        This is done by subtracting the mean of the weights from the weights themselves. This is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t"><span class="str">        done in-place. As softmax is translation invariant, this changes the logits but not the log</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t"><span class="str">        probs, and makes the model logits (slightly) more interpretable - when trying to understand</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t"><span class="str">        how components contribute to the logits, we'll be less misled by components that just add</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t"><span class="str">        something to every logit.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">        <span class="str">"""Fold the value biases into the output bias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t"><span class="str">        Because attention patterns add up to 1, the value biases always have a constant effect on a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t"><span class="str">        head's output. Further, as the outputs of each head in a layer add together, each head's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t"><span class="str">        value bias has a constant effect on the *layer's* output, which can make it harder to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t"><span class="str">        interpret the effect of any given head, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t"><span class="str">        associated with. We can factor this all into a single output bias to the layer, and make it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t"><span class="str">        easier to interpret the head's output. Formally, we take b_O_new = b_O_original +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t"><span class="str">        sum_head(b_V_head @ W_O_head).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">            <span class="com"># shape [head_index, d_head]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1829&#x202F;&#x219B;&#x202F;1832</span><span class="annotate long">line 1829 didn't jump to line 1832, because the condition on line 1829 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">                    <span class="nam">b_V</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">,</span> <span class="nam">repeats</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span> <span class="op">//</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">            <span class="com"># [head_index, d_head, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">            <span class="com"># [d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">b_O_original</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">            <span class="nam">folded_b_O</span> <span class="op">=</span> <span class="nam">b_O_original</span> <span class="op">+</span> <span class="op">(</span><span class="nam">b_V</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_O</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">folded_b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1843&#x202F;&#x219B;&#x202F;1846</span><span class="annotate long">line 1843 didn't jump to line 1846, because the condition on line 1843 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">    <span class="key">def</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">        <span class="str">"""Experimental method for managing queries, keys and values.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t"><span class="str">        As argued in [A Mathematical Framework for Transformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t"><span class="str">        Circuits](https://transformer-circuits.pub/2021/framework/index.html), queries, keys and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t"><span class="str">        values are somewhat arbitrary intermediate terms when computing with the low rank factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t"><span class="str">        matrices W_QK = W_Q @ W_K.T and W_OV = W_V @ W_O, and these matrices are the only thing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t"><span class="str">        determining head behaviour. But there are many ways to find a low rank factorization to a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t"><span class="str">        given matrix, and hopefully some of these are more interpretable than others! This method is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t"><span class="str">        one attempt, which makes all of the matrices have orthogonal rows or columns, W_O into a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t"><span class="str">        rotation and W_Q and W_K having the nth column in each having the same norm. The formula is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t"><span class="str">        $W_V = U @ S,W_O=Vh.T,W_Q=U@S.sqrt(),W_K=Vh@S.sqrt()$.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t"><span class="str">        More details:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t"><span class="str">        If W_OV = U @ S @ Vh.T in its singular value decomposition, (where S is in R^d_head not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t"><span class="str">        R^d_model, as W_OV is low rank), W_OV = (U @ S) @ (Vh.T) is an equivalent low rank</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t"><span class="str">        factorisation, where rows/columns of each matrix are orthogonal! So setting $W_V=US$ and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t"><span class="str">        $W_O=Vh.T$ works just as well. I *think* this is a more interpretable setup, because now</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t"><span class="str">        $W_O$ is just a rotation, and doesn't change the norm, so $z$ has the same norm as the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t"><span class="str">        result of the head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t"><span class="str">        For $W_QK = W_Q @ W_K.T$ we use the refactor $W_Q = U @ S.sqrt()$ and $W_K = Vh @ S.sqrt()$,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t"><span class="str">        which is also equivalent ($S==S.sqrt() @ S.sqrt()$ as $S$ is diagonal). Here we keep the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t"><span class="str">        matrices as having the same norm, since there's not an obvious asymmetry between the keys</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t"><span class="str">        and queries.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t"><span class="str">        Biases are more fiddly to deal with. For OV it's pretty easy - we just need (x @ W_V + b_V)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t"><span class="str">        @ W_O + b_O to be preserved, so we can set b_V' = 0. and b_O' = b_V @ W_O + b_O (note that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t"><span class="str">        b_V in R^{head_index x d_head} while b_O in R^{d_model}, so we need to sum b_V @ W_O along</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t"><span class="str">        the head_index dimension too).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t"><span class="str">        For QK it's messy - we need to preserve the bilinear form of (x @ W_Q + b_Q) * (y @ W_K +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t"><span class="str">        b_K), which is fairly messy. To deal with the biases, we concatenate them to W_Q and W_K to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t"><span class="str">        simulate a d_model+1 dimensional input (whose final coordinate is always 1), do the SVD</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t"><span class="str">        factorization on this effective matrix, then separate out into final weights and biases.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"You can't refactor the QK circuit when using rotary embeddings (as the QK matrix depends on the position of the query and key)"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">            <span class="com"># W_QK = W_Q @ W_K.T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">            <span class="com"># Concatenate biases to make a d_model+1 input dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t">            <span class="nam">W_Q_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">            <span class="nam">W_K_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">            <span class="nam">W_Q_eff_even</span><span class="op">,</span> <span class="nam">W_K_eff_even_T</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">                <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_Q_eff</span><span class="op">,</span> <span class="nam">W_K_eff</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">)</span><span class="op">.</span><span class="nam">make_even</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">pair</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">            <span class="nam">W_K_eff_even</span> <span class="op">=</span> <span class="nam">W_K_eff_even_T</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">            <span class="com"># W_OV = W_V @ W_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">            <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">            <span class="com"># Factors the bias to be consistent.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">            <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">            <span class="nam">b_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">            <span class="nam">effective_bias</span> <span class="op">=</span> <span class="nam">b_O</span> <span class="op">+</span> <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">                <span class="str">"head_index d_head, head_index d_head d_model -> d_model"</span><span class="op">,</span> <span class="nam">b_V</span><span class="op">,</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1931" href="#t1931">1931</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1932" href="#t1932">1932</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">effective_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t"><span class="key">class</span> <span class="nam">HookedTransformer</span><span class="op">(</span><span class="nam">HookedRootModule</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t">    <span class="str">"""Hooked Transformer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t"><span class="str">    Implements a full Transformer using the components :doc:`here &lt;transformer_lens.components>`,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"><span class="str">    with a :class:`transformer_lens.hook_points.HookPoint` on every interesting activation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t"><span class="str">    TransformerLens comes loaded with >50 GPT-style models. Typically you initialise it with one of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"><span class="str">    these via :meth:`from_pretrained`, although it can also be instantiated with randomly</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t"><span class="str">    initialized weights via :meth:`__init__`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="str">    Once you've initialized the model, a common next step is to test it can do the task you're</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str">    investigating. This can be done with :func:`transformer_lens.utils.test_prompt`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t">    <span class="nam">ln_final</span><span class="op">:</span> <span class="nam">nn</span><span class="op">.</span><span class="nam">Module</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t">        <span class="nam">cfg</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">HookedTransformerConfig</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">PreTrainedTokenizerBase</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">        <span class="str">"""Model initialization.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t"><span class="str">        Note that if you want to load the model from pretrained weights, you should use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t"><span class="str">        :meth:`from_pretrained` instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t"><span class="str">            cfg: The config to use for the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not provided, it is inferred from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t"><span class="str">                `cfg.tokenizer_name` or initialized to `None`. If `None`, then the model cannot be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t"><span class="str">                passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in cfg.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"><span class="str">                device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">        <span class="nam">super</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">__init__</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">133&#x202F;&#x219B;&#x202F;134</span><span class="annotate long">line 133 didn't jump to line 134, because the condition on line 133 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">                <span class="str">"Please pass in a config dictionary or HookedTransformerConfig object. If you want to load a "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">                <span class="str">"pretrained model, use HookedTransformer.from_pretrained() instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">set_tokenizer</span><span class="op">(</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">            <span class="com"># If we have a tokenizer name, we can load it from HuggingFace</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">145&#x202F;&#x219B;&#x202F;146</span><span class="annotate long">line 145 didn't jump to line 146, because the condition on line 145 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t">                    <span class="str">"%s tokenizer not loaded. Please load manually."</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">                <span class="com"># Hugging Face defaults to use_fast to True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">                <span class="nam">use_fast</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">                <span class="com"># Phi model's fast tokenizer does not support adding a BOS token, use_fast</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">                <span class="com"># should be False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">                <span class="key">if</span> <span class="str">"phi"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">155&#x202F;&#x219B;&#x202F;156</span><span class="annotate long">line 155 didn't jump to line 156, because the condition on line 155 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">                    <span class="nam">use_fast</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">                <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">set_tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">                    <span class="nam">AutoTokenizer</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">                        <span class="nam">add_bos_token</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t">                        <span class="nam">trust_remote_code</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">trust_remote_code</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">                        <span class="nam">use_fast</span><span class="op">=</span><span class="nam">use_fast</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t">                        <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">                    <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t">            <span class="com"># If no tokenizer name is provided, we assume we're training on an algorithmic task and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t">            <span class="com"># will pass in tokens directly. In this case, we don't need a tokenizer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">!=</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if d_vocab is not provided"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t">            <span class="key">if</span> <span class="nam">default_padding_side</span> <span class="op">!=</span> <span class="str">"right"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">173&#x202F;&#x219B;&#x202F;174</span><span class="annotate long">line 173 didn't jump to line 174, because the condition on line 173 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t">                    <span class="str">"default_padding_side is explictly given but ignored because tokenizer is not set."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span> <span class="op">=</span> <span class="nam">Embed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">PosEmbed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos, d__dictmodel]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_tokens</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_tokens</span> <span class="op">=</span> <span class="nam">HookPoint</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span> <span class="op">=</span> <span class="nam">nn</span><span class="op">.</span><span class="nam">ModuleList</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t">            <span class="op">[</span><span class="nam">TransformerBlock</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">block_index</span><span class="op">)</span> <span class="key">for</span> <span class="nam">block_index</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">192&#x202F;&#x219B;&#x202F;193</span><span class="annotate long">line 192 didn't jump to line 193, because the condition on line 192 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMSPre"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">194&#x202F;&#x219B;&#x202F;195</span><span class="annotate long">line 194 didn't jump to line 195, because the condition on line 194 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">197&#x202F;&#x219B;&#x202F;198</span><span class="annotate long">line 197 didn't jump to line 198, because the condition on line 197 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LNPre"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t">            <span class="com"># We've folded in LayerNorm weights, so just need the center + scale parts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">207&#x202F;&#x219B;&#x202F;211</span><span class="annotate long">line 207 didn't jump to line 211, because the condition on line 207 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">            <span class="com"># If it's None, don't create either layer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Invalid normalization_type passed in %s"</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span> <span class="op">=</span> <span class="nam">Unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">init_weights</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">            <span class="com"># We load the devices in a pipeline manner - the first device gets the embed and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">            <span class="com"># pos_embed layers and the first n_layers // n_devices blocks, the second gets the next</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">            <span class="com"># n_layers // n_devices blocks ... the last gets the last n_layers // n_devices blocks,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">            <span class="com"># the final normalization layer (if it exists) and the unembed layer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">        <span class="com"># Helper variable to store a small (10K-20K) dataset of training data. Empty by default, can</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">        <span class="com"># be loaded with load_sample_training_dataset</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">        <span class="com"># Gives each module a parameter with its name (relative to this root module)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">        <span class="com"># Needed for HookPoints to work</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">setup</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">    <span class="key">def</span> <span class="nam">check_hooks_to_add</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">        <span class="nam">hook_point</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">        <span class="nam">hook_point_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">        <span class="nam">hook</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">        <span class="nam">dir</span><span class="op">=</span><span class="str">"fwd"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">        <span class="nam">is_permanent</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">        <span class="nam">prepend</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"attn.hook_result"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_attn_result_hook is False"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="op">(</span><span class="str">"hook_q_input"</span><span class="op">,</span> <span class="str">"hook_k_input"</span><span class="op">,</span> <span class="str">"hook_v_input"</span><span class="op">)</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_split_qkv_input is False"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"mlp_in"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_hook_mlp_in is False"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t">        <span class="key">if</span> <span class="nam">hook_point_name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"attn_in"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Cannot add hook {hook_point_name} if use_attn_in is False"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">    <span class="key">def</span> <span class="nam">input_to_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>  <span class="com"># residual</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>  <span class="com"># tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>  <span class="com"># shortformer_pos_embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">        <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>  <span class="com"># attention_mask [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">        <span class="str">"""Convert input to first residual stream.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t"><span class="str">            input (Union[str, List[str], Int[torch.Tensor, "batch pos"]]): The input to the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t"><span class="str">            padding_side ([Literal["left", "right"], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t"><span class="str">                multiple strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t"><span class="str">            past_kv_cache (HookedTransformerKeyValueCache, optional): If passed, we're doing caching</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t"><span class="str">                and attention_mask will be stored in the cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span> <span class="key">or</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t">            <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">            <span class="com"># This is only intended to support passing in a single string</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">294&#x202F;&#x219B;&#x202F;296</span><span class="annotate long">line 294 didn't jump to line 296, because the condition on line 294 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">            <span class="com"># If tokens are a rank 1 tensor, add a dummy batch dimension to avoid things breaking.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">[</span><span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span><span class="op">.</span><span class="nam">type</span> <span class="op">!=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">            <span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">padding_side</span> <span class="op">==</span> <span class="str">"left"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">            <span class="key">or</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">            <span class="key">or</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">            <span class="com"># This means we need to have an explicit attention mask.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">            <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">                <span class="com"># If the padding side is left or we are using caching, we need to compute the attention</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">                <span class="com"># mask for the adjustment of absolute positional embeddings and attention masking so</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">                <span class="com"># that pad tokens are not attended.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">                <span class="key">if</span> <span class="nam">prepend_bos</span> <span class="key">is</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">                    <span class="nam">prepend_bos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">                <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_attention_mask</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">            <span class="key">assert</span> <span class="nam">attention_mask</span><span class="op">.</span><span class="nam">shape</span> <span class="op">==</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">                <span class="str">f"Attention mask shape {attention_mask.shape} does not match tokens shape "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">                <span class="str">f"{tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">            <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">attention_mask</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">            <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">                <span class="com"># past_kv_cache is not None, so we're doing caching.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">                <span class="com"># We need to extend the previous attention_mask.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">                <span class="com"># Update the past_kv_cache with the new attention_mask (unless it's frozen)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">                <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">past_kv_cache</span><span class="op">.</span><span class="nam">append_attention_mask</span><span class="op">(</span><span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">            <span class="com"># We separate this case from for computational efficiency.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">            <span class="nam">attention_mask</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">        <span class="com"># If we're doing caching, then we reuse keys and values from previous runs, as that's the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">        <span class="com"># only way that past activations will affect the final logits. The cache contains those so</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="com"># we don't need to recompute them. This is useful for generating text. As we have absolute</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="com"># positional encodings, to implement this we have a `pos_offset` variable, defaulting to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="com"># zero, which says to offset which positional encodings are used (cached keys and values</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">        <span class="com"># were calculated with their own positional encodings).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">        <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">            <span class="nam">pos_offset</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">            <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">                <span class="nam">cached_batch_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">                <span class="nam">cache_ctx_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">                <span class="nam">num_heads_in_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">                <span class="nam">d_head_in_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">            <span class="op">)</span> <span class="op">=</span> <span class="nam">past_kv_cache</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">.</span><span class="nam">past_keys</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">            <span class="key">assert</span> <span class="nam">cached_batch_size</span> <span class="op">==</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">345&#x202F;&#x219B;&#x202F;348</span><span class="annotate long">line 345 didn't jump to line 348, because the condition on line 345 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">                <span class="key">assert</span> <span class="nam">num_heads_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">                <span class="key">assert</span> <span class="nam">num_heads_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">            <span class="key">assert</span> <span class="nam">d_head_in_cache</span> <span class="op">==</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">            <span class="nam">pos_offset</span> <span class="op">=</span> <span class="nam">cache_ctx_length</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_tokens</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_tokens</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">        <span class="nam">embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"standard"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">            <span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span> <span class="op">+</span> <span class="nam">pos_embed</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">            <span class="com"># If we're using shortformer style attention, we don't add the positional embedding to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">            <span class="com"># the residual stream. See HookedTransformerConfig for details</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">            <span class="nam">pos_embed</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="nam">pos_embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">            <span class="com"># Rotary doesn't use positional embeddings, instead they're applied when dot producting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">            <span class="com"># keys and queries. See HookedTransformerConfig for details</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"alibi"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">373&#x202F;&#x219B;&#x202F;378</span><span class="annotate long">line 373 didn't jump to line 378, because the condition on line 373 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">            <span class="com"># ALiBi does not add positional embeddings to word embeddings,instead it biases QK attention scores.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">            <span class="nam">residual</span> <span class="op">=</span> <span class="nam">embed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">            <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">                <span class="str">f"Invalid positional_embedding_type passed in {self.cfg.positional_embedding_type}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">        <span class="key">return</span> <span class="nam">residual</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">shortformer_pos_embed</span><span class="op">,</span> <span class="nam">attention_mask</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"logits"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Loss</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"loss"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Loss</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"both"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">None</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">        <span class="nam">loss_per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="nam">start_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">        <span class="nam">shortformer_pos_embed</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># [batch pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="nam">stop_at_layer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="nam">past_kv_cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">HookedTransformerKeyValueCache</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">        <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="nam">Loss</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">        <span class="str">"""Forward Pass.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t"><span class="str">        Input is either a batch of tokens ([batch, pos]) or a text string, a string is automatically</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t"><span class="str">        tokenized to a batch of a single element. The prepend_bos flag only applies when inputting a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t"><span class="str">        text string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t"><span class="str">        Note that loss is the standard "predict the next token" cross-entropy loss for GPT-2 style</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t"><span class="str">        language models - if you want a custom loss function, the recommended behaviour is returning</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t"><span class="str">        the logits and then applying your custom loss function.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t"><span class="str">            return_type Optional[str]: The type of output to return. Can be one of: None (return</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t"><span class="str">                nothing, don't calculate logits), 'logits' (return logits), 'loss' (return</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t"><span class="str">                cross-entropy loss), 'both' (return logits and loss).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t"><span class="str">            loss_per_token bool: Whether to return the (next token prediction) loss per token (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t"><span class="str">                or average (False). Average loss is a scalar (averaged over position *and* batch),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t"><span class="str">                per-token loss is a tensor ([batch, position-1]) - position-1 because we're</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t"><span class="str">                predicting the next token, and there's no specified next token for the final token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t"><span class="str">                Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t"><span class="str">            prepend_bos Optional[bool]: Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t"><span class="str">                otherwise. (Even for models not explicitly trained with a prepended BOS token, heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t"><span class="str">                often use the first position as a resting position and accordingly lose information</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t"><span class="str">                from the first token, so this empirically seems to give better results.) Pass True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t"><span class="str">                or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t"><span class="str">            padding_side Optional[Literal["left", "right"]]: Overrides self.tokenizer.padding_side.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t"><span class="str">                Specifies which side to pad on when tokenizing multiple strings of different</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t"><span class="str">                lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t"><span class="str">            start_at_layer Optional[int]: If not None, start the forward pass at the specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t"><span class="str">                layer. Requires input to be the residual stream before the specified layer with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t"><span class="str">                shape [batch, pos, d_model]. Inclusive - ie, start_at_layer = 0 skips the embedding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t"><span class="str">                then runs the rest of the model. Supports negative indexing. start_at_layer = -1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t"><span class="str">                only runs the final block and the unembedding. Defaults to None (run the full</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t"><span class="str">                model).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t"><span class="str">            tokens: Optional[Int[torch.Tensor, "batch pos"]]: Tokenized input. Only use if</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t"><span class="str">                start_at_layer is not None and return type is "loss" or "both".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t"><span class="str">            shortformer_pos_embed: Optional[Float[torch.Tensor, "batch pos d_model"]]: Positional</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t"><span class="str">                embedding for shortformer models. Only use if start_at_layer is not None and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t"><span class="str">                self.cfg.positional_embedding_type == "shortformer".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t"><span class="str">            attention_mask: Optional[torch.Tensor]: Override the attention mask used to ignore</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t"><span class="str">                padded tokens. If start_at_layer is not None and (self.tokenizer.padding_side ==</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t"><span class="str">                "left" or past_kv_cache is not None), this should be passed as the attention mask</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t"><span class="str">                is not computed automatically. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t"><span class="str">            stop_at_layer Optional[int]: If not None, stop the forward pass at the specified layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t"><span class="str">                Exclusive - ie, stop_at_layer = 0 will only run the embedding layer, stop_at_layer =</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t"><span class="str">                1 will run the embedding layer and the first transformer block, etc. Supports</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t"><span class="str">                negative indexing. Useful for analysis of intermediate layers, eg finding neuron</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t"><span class="str">                activations in layer 3 of a 24 layer model. Defaults to None (run the full model).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t"><span class="str">                If not None, we return the last residual stream computed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t"><span class="str">            past_kv_cache Optional[HookedTransformerKeyValueCache]: If not None, keys and values</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t"><span class="str">                will be stored for every attention head (unless the cache is frozen). If there are</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t"><span class="str">                keys and values already in the cache, these will be prepended to the keys and values</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t"><span class="str">                for the new input, so that the new tokens can pay attention to previous tokens. This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t"><span class="str">                is useful for generating text, because we don't need to repeat computation for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t"><span class="str">                tokens that have already been through the model. Also caches attention_mask so</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t"><span class="str">                previous tokens are masked correctly (unless frozen). Padding should be ignored in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t"><span class="str">                all cases, so it's okay to eg. pass in left padded tokens twice in a row.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t"><span class="str">                Warning: Don't accidentally prepend_bos to the second half of a prompt.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t"><span class="str">                Defaults to None (don't use caching).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">            <span class="key">if</span> <span class="nam">start_at_layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">                <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">                    <span class="nam">residual</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">                <span class="op">)</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">input_to_embed</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">                    <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">                    <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">                    <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">=</span><span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">                    <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">                <span class="key">assert</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">            <span class="key">if</span> <span class="nam">start_at_layer</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">                <span class="nam">start_at_layer</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">            <span class="com"># If we explicitly want to start or stop at a layer, we only iterate through the blocks</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">            <span class="com"># between those indices. Note that start_at_layer is inclusive and stop_at_layer is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="com"># exclusive.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">            <span class="com"># Eg: start_at_layer==None + stop_at_layer==0 means to only run the embed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="com"># Eg: start_at_layer==3 + stop_at_layer==-1 means to run from layer 3 until the end of the PENULTIMATE layer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="nam">blocks_and_idxs</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">zip</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">            <span class="key">for</span> <span class="nam">i</span><span class="op">,</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">blocks_and_idxs</span><span class="op">[</span><span class="nam">start_at_layer</span><span class="op">:</span><span class="nam">stop_at_layer</span><span class="op">]</span><span class="op">:</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">                <span class="com"># Note that each block includes skip connections, so we don't need</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">                <span class="com"># residual + block(residual)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">                <span class="com"># If we're using multiple GPUs, we need to send the residual and shortformer_pos_embed to the correct GPU</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">residual</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">                <span class="key">if</span> <span class="nam">shortformer_pos_embed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span> <span class="op">=</span> <span class="nam">shortformer_pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">block</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">                    <span class="nam">residual</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">                    <span class="com"># Cache contains a list of HookedTransformerKeyValueCache objects, one for each</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">                    <span class="com"># block</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">                    <span class="nam">past_kv_cache_entry</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span> <span class="key">if</span> <span class="nam">past_kv_cache</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">                    <span class="nam">shortformer_pos_embed</span><span class="op">=</span><span class="nam">shortformer_pos_embed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">                    <span class="nam">attention_mask</span><span class="op">=</span><span class="nam">attention_mask</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">                <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_layer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">                <span class="com"># When we stop at an early layer, we end here rather than doing further computation</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">                <span class="key">return</span> <span class="nam">residual</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">                <span class="nam">residual</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span><span class="op">(</span><span class="nam">residual</span><span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">                <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">                <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">(</span><span class="nam">residual</span><span class="op">)</span>  <span class="com"># [batch, pos, d_vocab]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">594&#x202F;&#x219B;&#x202F;595</span><span class="annotate long">line 594 didn't jump to line 595, because the condition on line 594 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">*</span> <span class="nam">F</span><span class="op">.</span><span class="nam">tanh</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">                <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"logits"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">                    <span class="key">return</span> <span class="nam">logits</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">                        <span class="nam">tokens</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"tokens must be passed in if return_type is 'loss' or 'both'"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">                    <span class="nam">loss</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">loss_fn</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">,</span> <span class="nam">per_token</span><span class="op">=</span><span class="nam">loss_per_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">                    <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"loss"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">605&#x202F;&#x219B;&#x202F;607</span><span class="annotate long">line 605 didn't jump to line 607, because the condition on line 605 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">                        <span class="key">return</span> <span class="nam">loss</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">                    <span class="key">elif</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"both"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">                        <span class="key">return</span> <span class="nam">Output</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">loss</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">                        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">f"Invalid return_type passed in: {return_type}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">                        <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="key">def</span> <span class="nam">loss_fn</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="nam">logits</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="nam">per_token</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">        <span class="str">"""Wrapper around `utils.lm_cross_entropy_loss`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t"><span class="str">        Used in forward() with return_type=="loss" or "both".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">        <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span> <span class="op">!=</span> <span class="nam">logits</span><span class="op">.</span><span class="nam">device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">624&#x202F;&#x219B;&#x202F;625</span><span class="annotate long">line 624 didn't jump to line 625, because the condition on line 624 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">logits</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">        <span class="key">return</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">lm_cross_entropy_loss</span><span class="op">(</span><span class="nam">logits</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">,</span> <span class="nam">per_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">True</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Output</span><span class="op">,</span> <span class="nam">ActivationCache</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">    <span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">False</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Output</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">        <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">return_cache_object</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">        <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">            <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">            <span class="nam">Loss</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">            <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_vocab"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Loss</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">        <span class="nam">Union</span><span class="op">[</span><span class="nam">ActivationCache</span><span class="op">,</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">        <span class="str">"""Wrapper around `run_with_cache` in HookedRootModule.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t"><span class="str">        If return_cache_object is True, this will return an ActivationCache object, with a bunch of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t"><span class="str">        useful HookedTransformer specific methods, otherwise it will return a dictionary of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t"><span class="str">        activations as in HookedRootModule.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">        <span class="nam">out</span><span class="op">,</span> <span class="nam">cache_dict</span> <span class="op">=</span> <span class="nam">super</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">            <span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">        <span class="key">if</span> <span class="nam">return_cache_object</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">660&#x202F;&#x219B;&#x202F;664</span><span class="annotate long">line 660 didn't jump to line 664, because the condition on line 660 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="nam">ActivationCache</span><span class="op">(</span><span class="nam">cache_dict</span><span class="op">,</span> <span class="nam">self</span><span class="op">,</span> <span class="nam">has_batch_dim</span><span class="op">=</span><span class="key">not</span> <span class="nam">remove_batch_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">            <span class="key">return</span> <span class="nam">out</span><span class="op">,</span> <span class="nam">cache</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">            <span class="key">return</span> <span class="nam">out</span><span class="op">,</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">        <span class="str">"""Set the tokenizer to use for this model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t"><span class="str">            tokenizer (PreTrainedTokenizer): a pretrained HuggingFace tokenizer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t"><span class="str">            default_padding_side (str): "right" or "left", which side to pad on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">PreTrainedTokenizerBase</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"{type(tokenizer)} is not a supported tokenizer, please use PreTrainedTokenizer or PreTrainedTokenizerFast"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">        <span class="key">assert</span> <span class="nam">default_padding_side</span> <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">            <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">            <span class="str">"left"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">        <span class="op">]</span><span class="op">,</span> <span class="str">f"padding_side must be 'right' or 'left', got {default_padding_side}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">        <span class="com"># Use a tokenizer that is initialized with add_bos_token=True as the default tokenizer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">        <span class="com"># Such a tokenizer should be set as the default tokenizer because the tokenization of some</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">        <span class="com"># tokenizers like LlamaTokenizer are different when bos token is automatically/manually</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">        <span class="com"># prepended, and add_bos_token cannot be dynamically controlled after initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">        <span class="com"># (https://github.com/huggingface/transformers/issues/25886).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">        <span class="nam">tokenizer_with_bos</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_tokenizer_with_bos</span><span class="op">(</span><span class="nam">tokenizer</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="op">=</span> <span class="nam">tokenizer_with_bos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">padding_side</span> <span class="op">=</span> <span class="nam">default_padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">        <span class="com"># Some tokenizers doesn't automatically prepend the BOS token even when they are initialized</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t">        <span class="com"># with add_bos_token=True. Therefore, we need this information to dynamically control prepend_bos.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">encode</span><span class="op">(</span><span class="str">""</span><span class="op">)</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">701&#x202F;&#x219B;&#x202F;702</span><span class="annotate long">line 701 didn't jump to line 702, because the condition on line 701 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span> <span class="op">=</span> <span class="str">"&lt;|endoftext|>"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">pad_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">705&#x202F;&#x219B;&#x202F;706</span><span class="annotate long">line 705 didn't jump to line 706, because the condition on line 705 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">bos_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">        <span class="com"># Infer vocab size from tokenizer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span> <span class="op">=</span> <span class="nam">max</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">vocab</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">+</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab_out</span> <span class="op">==</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">        <span class="nam">truncate</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">        <span class="str">"""Converts a string to a tensor of tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t"><span class="str">        If prepend_bos is True, prepends the BOS token to the input - this is recommended when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t"><span class="str">        creating a sequence of tokens to be input to a model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t"><span class="str">        Gotcha: prepend_bos prepends a beginning of string token. This is a recommended default when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t"><span class="str">        inputting a prompt to the model as the first token is often treated weirdly, but should only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t"><span class="str">        be done at the START of the prompt. Make sure to turn it off if you're looking at the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t"><span class="str">        tokenization of part of the prompt! (Note: some models eg GPT-2 were not trained with a BOS</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t"><span class="str">        token, others (OPT and my models) were)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t"><span class="str">        Gotcha2: Tokenization of a string depends on whether there is a preceding space and whether</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t"><span class="str">        the first letter is capitalized. It's easy to shoot yourself in the foot here if you're not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t"><span class="str">        careful!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t"><span class="str">            input (Union[str, List[str]]): The input to tokenize.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t"><span class="str">                multiple strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t"><span class="str">            move_to_device (bool): Whether to move the output tensor of tokens to the device the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t"><span class="str">                model lives on. Defaults to True truncate (bool): If the output tokens are too long,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t"><span class="str">                whether to truncate the output tokens to the model's max context window. Does nothing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t"><span class="str">                for shorter inputs. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">,</span> <span class="str">"Cannot use to_tokens without a tokenizer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Set the tokenizer for the model by calling set_tokenizer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">                <span class="com"># We want to prepend bos but the tokenizer doesn't automatically do it, so we add it manually</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">                <span class="nam">input</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_input_with_manually_prepended_bos</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">input</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">                <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">                <span class="nam">return_tensors</span><span class="op">=</span><span class="str">"pt"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">                <span class="nam">padding</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">                <span class="nam">truncation</span><span class="op">=</span><span class="nam">truncate</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">                <span class="nam">max_length</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_ctx</span> <span class="key">if</span> <span class="nam">truncate</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="str">"input_ids"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">tokenizer_prepends_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">                <span class="com"># We don't want to prepend bos but the tokenizer does it automatically, so we remove it manually</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_tokens_with_bos_removed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">            <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">            <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_string</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">            <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">            <span class="nam">List</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">        <span class="str">"""Tokens to String(s).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t"><span class="str">        Converts a tensor of tokens to a string (if rank 1) or a list of strings (if rank 2).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t"><span class="str">        Accepts lists of tokens and numpy arrays as inputs too (and converts to tensors internally)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">,</span> <span class="str">"Cannot use to_string without a tokenizer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">            <span class="com"># We allow lists to be input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">        <span class="com"># I'm not sure what exactly clean_up_tokenization_spaces does, but if</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="com"># it's set, then tokenization is no longer invertible, and some tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">        <span class="com"># with a bunch of whitespace get collapsed together</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">2</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">batch_decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">        <span class="key">elif</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">&lt;=</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">807&#x202F;&#x219B;&#x202F;810</span><span class="annotate long">line 807 didn't jump to line 810, because the condition on line 807 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid shape passed in: {tokens.shape}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_str_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">            <span class="nam">list</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">List</span><span class="op">[</span><span class="nam">List</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">        <span class="str">"""Map text, a list of text or tokens to a list of tokens as strings.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t"><span class="str">        Gotcha: prepend_bos prepends a beginning of string token. This is a recommended default when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t"><span class="str">        inputting a prompt to the model as the first token is often treated weirdly, but should only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t"><span class="str">        be done at the START of the prompt. If prepend_bos=None is passed, it implies the usage of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t"><span class="str">        self.cfg.default_prepend_bos which is set to True unless specified otherwise. Therefore,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t"><span class="str">        make sure to locally turn it off by passing prepend_bos=False if you're looking at the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t"><span class="str">        tokenization of part of the prompt! (Note: some models eg GPT-2 were not trained with a BOS</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t"><span class="str">        token, others (OPT and my models) were)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t"><span class="str">        Gotcha2: Tokenization of a string depends on whether there is a preceding space and whether</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t"><span class="str">        the first letter is capitalized. It's easy to shoot yourself in the foot here if you're not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t"><span class="str">        careful!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t"><span class="str">        Gotcha3: If passing a string that exceeds the model's context length (model.cfg.n_ctx), it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t"><span class="str">        will be truncated.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t"><span class="str">            input (Union[str, list, torch.Tensor]): The input - either a string or a tensor of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t"><span class="str">                tokens. If tokens, should be a tensor of shape [pos] or [1, pos].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t"><span class="str">            str_tokens: List of individual tokens as strings</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">            <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">                <span class="key">return</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">                    <span class="nam">map</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">                        <span class="key">lambda</span> <span class="nam">tokens</span><span class="op">:</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_str_tokens</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">                        <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">                <span class="op">)</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">                    <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">                <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">                <span class="com"># Gemma tokenizer expects a batch dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">                <span class="key">if</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">name_or_path</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">873&#x202F;&#x219B;&#x202F;874</span><span class="annotate long">line 873 didn't jump to line 874, because the condition on line 873 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>  <span class="com"># Get rid of a trivial batch dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">                <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">                    <span class="com"># Don't pass dimensionless tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">np</span><span class="op">.</span><span class="nam">ndarray</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">884&#x202F;&#x219B;&#x202F;894</span><span class="annotate long">line 884 didn't jump to line 894, because the condition on line 884 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>  <span class="com"># Get rid of a trivial batch dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">                <span class="key">if</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">                    <span class="com"># Don't pass dimensionless tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">                    <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">expand_dims</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">axis</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">                    <span class="nam">tokens</span><span class="op">.</span><span class="nam">ndim</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">f"Invalid tokens input to to_str_tokens, has shape: {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid input type to to_str_tokens: {type(input)}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">            <span class="nam">str_tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">batch_decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">clean_up_tokenization_spaces</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">            <span class="key">return</span> <span class="nam">str_tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_single_token</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">string</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">        <span class="str">"""Map a string that makes up a single token to the id for that token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t"><span class="str">        Raises an error for strings that are not a single token! If uncertain use to_tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">        <span class="com"># We use the to_tokens method, do not append a BOS token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">        <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">string</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">.</span><span class="nam">squeeze</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">        <span class="com"># If token shape is non-empty, raise error</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">token</span><span class="op">.</span><span class="nam">shape</span><span class="op">,</span> <span class="str">f"Input string: {string} is not a single token!"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">        <span class="key">return</span> <span class="nam">token</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">    <span class="key">def</span> <span class="nam">to_single_str_token</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">int_token</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">        <span class="com"># Gives the single token corresponding to an int in string form</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">        <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">int_token</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">        <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_str_tokens</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="op">[</span><span class="nam">int_token</span><span class="op">]</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">        <span class="key">assert</span> <span class="nam">len</span><span class="op">(</span><span class="nam">token</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">        <span class="key">return</span> <span class="nam">cast</span><span class="op">(</span><span class="nam">str</span><span class="op">,</span> <span class="nam">token</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_token_position</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">        <span class="nam">single_token</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">int</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">        <span class="nam">mode</span><span class="op">=</span><span class="str">"first"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">        <span class="str">"""Get the position of a single_token in a string or sequence of tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t"><span class="str">        Raises an error if the token is not present.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t"><span class="str">        Gotcha: If you're inputting a string, it'll automatically be tokenized. Be careful about the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t"><span class="str">        setting for prepend_bos! When a string is input to the model, a BOS (beginning of sequence)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t"><span class="str">        token is prepended by default when the string is tokenized because</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t"><span class="str">        self.cfg.default_prepend_bos is set to True unless specified otherwise. But this should only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t"><span class="str">        be done at the START of the input, not when inputting part of the prompt. If you're getting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t"><span class="str">        weird off-by-one errors, check carefully for what the setting should be!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t"><span class="str">            single_token (Union[str, int]): The token to search for. Can</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t"><span class="str">                be a token index, or a string (but the string must correspond to a single token).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t"><span class="str">            input (Union[str, torch.Tensor]): The sequence to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t"><span class="str">                search in. Can be a string or a rank 1 tensor of tokens or a rank 2 tensor of tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t"><span class="str">                with a dummy batch dimension.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t"><span class="str">            mode (str, optional): If there are multiple matches, which match to return. Supports</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t"><span class="str">                "first" or "last". Defaults to "first".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t"><span class="str">                the BOS token to the input (only applies when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos which is set to True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t"><span class="str">                otherwise. Pass True or False to locally override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">            <span class="com"># If the input is a string, convert to tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">)</span> <span class="op">==</span> <span class="num">2</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">            <span class="com"># If the tokens have shape [1, seq_len], flatten to [seq_len]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">                <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="op">==</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"If tokens are rank two, they must have shape [1, seq_len], not {tokens.shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">single_token</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">            <span class="com"># If the single token is a string, convert to an integer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="nam">single_token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">single_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">single_token</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">968&#x202F;&#x219B;&#x202F;969</span><span class="annotate long">line 968 didn't jump to line 969, because the condition on line 968 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">            <span class="nam">single_token</span> <span class="op">=</span> <span class="nam">single_token</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">        <span class="nam">indices</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">len</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">tokens</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="nam">tokens</span> <span class="op">==</span> <span class="nam">single_token</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">        <span class="key">assert</span> <span class="nam">len</span><span class="op">(</span><span class="nam">indices</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span><span class="op">,</span> <span class="str">"The token does not occur in the prompt"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"first"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">            <span class="key">return</span> <span class="nam">indices</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"last"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">975&#x202F;&#x219B;&#x202F;978</span><span class="annotate long">line 975 didn't jump to line 978, because the condition on line 975 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="key">return</span> <span class="nam">indices</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be 'first' or 'last', not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">    <span class="key">def</span> <span class="nam">tokens_to_residual_directions</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">        <span class="nam">tokens</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">""</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">            <span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">        <span class="str">"""Map tokens to a tensor with the unembedding vector for those tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t"><span class="str">        I.e. the vector in the residual stream that we dot with to the get the logit for that token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t"><span class="str">        WARNING: If you use this without folding in LayerNorm, the results will be misleading and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t"><span class="str">        may be incorrect, as the LN weights change the unembed map. This is done automatically with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t"><span class="str">        the fold_ln flag on from_pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t"><span class="str">        WARNING 2: LayerNorm scaling will scale up or down the effective direction in the residual</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t"><span class="str">        stream for each output token on any given input token position.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t"><span class="str">        ActivationCache.apply_ln_to_stack will apply the appropriate scaling to these directions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t"><span class="str">            tokens (Union[str, int, torch.Tensor]): The token(s). If a single token, can be a single</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t"><span class="str">                element tensor, an integer, or string. If string, will be mapped to a single token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t"><span class="str">                using to_single_token, and an error raised if it's multiple tokens. The method also</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t"><span class="str">                works for a batch of input tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t"><span class="str">            residual_direction torch.Tensor: The unembedding vector for the token(s), a stack of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t"><span class="str">                [d_model] tensor.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">numel</span><span class="op">(</span><span class="op">)</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">            <span class="com"># If the tokens are a tensor, and have more than one element, assume they are a batch of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="com"># tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="nam">residual_directions</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_U</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="nam">tokens</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="nam">residual_directions</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">                <span class="nam">residual_directions</span><span class="op">,</span> <span class="str">"d_model ... -> ... d_model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_directions</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">            <span class="com"># Otherwise there is a single token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1026&#x202F;&#x219B;&#x202F;1027</span><span class="annotate long">line 1026 didn't jump to line 1027, because the condition on line 1026 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_single_token</span><span class="op">(</span><span class="nam">tokens</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1028&#x202F;&#x219B;&#x202F;1029</span><span class="annotate long">line 1028 didn't jump to line 1029, because the condition on line 1028 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">            <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span> <span class="key">and</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">numel</span><span class="op">(</span><span class="op">)</span> <span class="op">==</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1030&#x202F;&#x219B;&#x202F;1033</span><span class="annotate long">line 1030 didn't jump to line 1033, because the condition on line 1030 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">                <span class="nam">token</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">item</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Invalid token type: {type(tokens)}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">            <span class="nam">residual_direction</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_U</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="nam">token</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">            <span class="key">return</span> <span class="nam">residual_direction</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">    <span class="key">def</span> <span class="nam">to</span><span class="op">(</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">        <span class="nam">device_or_dtype</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">        <span class="nam">print_details</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">        <span class="key">return</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">move_to_and_update_config</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">device_or_dtype</span><span class="op">,</span> <span class="nam">print_details</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">    <span class="key">def</span> <span class="nam">cuda</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">        <span class="str">"""Wrapper around cuda that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"cuda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">    <span class="key">def</span> <span class="nam">cpu</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">        <span class="str">"""Wrapper around cuda that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"cpu"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">    <span class="key">def</span> <span class="nam">mps</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">        <span class="str">"""Wrapper around mps that also changes `self.cfg.device`."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="str">"mps"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">    <span class="key">def</span> <span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">hook_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">hook_pos_embed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">        <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="str">"ln_final"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span> <span class="op">-</span> <span class="num">1</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span><span class="op">,</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">enumerate</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">            <span class="nam">block</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="nam">i</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">        <span class="nam">cls</span><span class="op">:</span> <span class="nam">Type</span><span class="op">[</span><span class="nam">T</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">        <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">        <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">        <span class="nam">hf_model</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">AutoModelForCausalLM</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">        <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">        <span class="nam">tokenizer</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">PreTrainedTokenizerBase</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">        <span class="nam">move_to_device</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="str">"float32"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">        <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">T</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">        <span class="str">"""Load in a Pretrained Model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t"><span class="str">        Load in pretrained model weights to the HookedTransformer format and optionally to do some</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t"><span class="str">        processing to make the model easier to interpret. Currently supports loading from most</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t"><span class="str">        autoregressive HuggingFace models (``gpt2``, ``neo``, ``gptj``, ``opt``...) and from a range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t"><span class="str">        of toy models and SoLU models trained by Neel Nanda. The full list is available in the docs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t"><span class="str">        under :doc:`model properties&lt;/generated/model_properties_table>`. Also supports loading from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t"><span class="str">        a checkpoint for checkpointed models (currently, models trained by NeelNanda and the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t"><span class="str">        stanford-crfm models (using parameters ``checkpoint_index`` and ``checkpoint_value``).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t"><span class="str">        See :meth:`load_and_process_state_dict` for details on the processing (folding layer norm,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t"><span class="str">        centering the unembedding and centering the writing weights).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t"><span class="str">            model_name: The model name - must be an element of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t"><span class="str">                :const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` or an alias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t"><span class="str">                of one. The full list of available models can be found in the docs under :doc:`model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t"><span class="str">                properties&lt;/generated/model_properties_table>`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t"><span class="str">            fold_ln: Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t"><span class="str">                `LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t"><span class="str">                &lt;https://wandb.ai/wandb_fc/LayerNorm/reports/Layer-Normalization-in-Pytorch-With-Examples---VmlldzoxMjk5MTk1>`_</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t"><span class="str">                is a common regularization technique used in transformers. Unlike BatchNorm, it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t"><span class="str">                cannot be turned off at inference time, as it significantly alters the mathematical</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t"><span class="str">                function implemented by the transformer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t"><span class="str">                When `fold_ln` is set to True, LayerNorm (with weights :math:`w_{ln}` and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t"><span class="str">                :math:`b_{ln}`) followed by a linear layer (:math:`W + b`) is optimized to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t"><span class="str">                LayerNormPre (just centering &amp; normalizing) followed by a new linear layer with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t"><span class="str">                :math:`W_{eff} = w[:, \text{None}] * W` (element-wise multiplication) and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t"><span class="str">                :math:`b_{eff} = b + b_{ln} @ W`. This transformation is computationally equivalent</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t"><span class="str">                and simplifies the model's interpretability. It essentially merges LayerNorm weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t"><span class="str">                into the subsequent linear layer's weights, which is handled by HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t"><span class="str">                when loading pre-trained weights. Set `fold_ln` to False when loading a state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t"><span class="str">                if you wish to turn this off.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t"><span class="str">                Mathematically, LayerNorm is defined as follows:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t"><span class="str">                .. math::</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t"><span class="str">                    x_1 &amp;= x_0 - \\text{mean}(x_0)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t"><span class="str">                    x_2 &amp;= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t"><span class="str">                    x_3 &amp;= x_2 \\cdot w</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t"><span class="str">                    x_4 &amp;= x_3 + b</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t"><span class="str">                For further details, refer to `this document</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t"><span class="str">                &lt;https://transformer-circuits.pub/2021/framework/index.html#:~:text=Handling%20Layer%20Normalization>`_.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t"><span class="str">            center_writing_weights: Whether to center weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t"><span class="str">                writing to the residual stream (ie set mean to be zero). Due to LayerNorm this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t"><span class="str">                doesn't change the computation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t"><span class="str">                A related idea to folding layernorm (``fold_ln``) - *every* component reading an</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t"><span class="str">                input from the residual stream is preceded by a LayerNorm, which means that the mean</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t"><span class="str">                of a residual stream vector (ie the component in the direction of all ones) never</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t"><span class="str">                matters. This means we can remove the all ones component of weights and biases whose</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t"><span class="str">                output *writes* to the residual stream. Mathematically, ``W_writing -=</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t"><span class="str">                W_writing.mean(dim=1, keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t"><span class="str">            center_unembed: Whether to center W_U (ie set mean</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t"><span class="str">                to be zero). Softmax is translation invariant so this doesn't affect log probs or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t"><span class="str">                loss, but does change logits.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t"><span class="str">                The logits are fed into a softmax. Softmax is translation invariant (eg, adding 1 to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t"><span class="str">                every logit doesn't change the output), so we can simplify things by setting the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t"><span class="str">                mean of the logits to be zero. This is equivalent to setting the mean of every</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t"><span class="str">                output vector of ``W_U`` to zero. In code, ``W_U -= W_U.mean(dim=-1,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t"><span class="str">                keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices: Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t"><span class="str">            checkpoint_index: If loading from a checkpoint, the index of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t"><span class="str">                the checkpoint to load.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t"><span class="str">            checkpoint_value: If loading from a checkpoint, the value of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t"><span class="str">                the checkpoint to load, ie the step or token number (each model has checkpoints</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t"><span class="str">                labelled with exactly one of these). E.g. ``1000`` for a checkpoint taken at step</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t"><span class="str">                1000 or after 1000 tokens. If `checkpoint_index` is also specified, this will be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t"><span class="str">                ignored.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t"><span class="str">            hf_model: If you have already loaded in the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t"><span class="str">                HuggingFace model, you can pass it in here rather than needing to recreate the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t"><span class="str">                object. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t"><span class="str">            device: The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t"><span class="str">                default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t"><span class="str">            n_devices: The number of devices to split the model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t"><span class="str">                across. Defaults to 1. If greater than 1, `device` must be cuda.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t"><span class="str">                provided, it is inferred from cfg.tokenizer_name or initialized to None. If None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t"><span class="str">                then the model cannot be passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t"><span class="str">                cfg. device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t"><span class="str">            fold_value_biases: Each attention head has a value bias. Values are averaged to create</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t"><span class="str">                mixed values (``z``), weighted by the attention pattern, but as the bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t"><span class="str">                constant, its contribution to ``z`` is exactly the same. The output of a head is ``z</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t"><span class="str">                @ W_O``, and so the value bias just linearly adds to the output of the head. This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t"><span class="str">                means that the value bias of a head has nothing to do with the head, and is just a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t"><span class="str">                constant added to the attention layer outputs. We can take the sum across these and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t"><span class="str">                b_O to get an "effective bias" for the layer. In code, we set ``b_V=0``. and ``b_O =</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t"><span class="str">                (b_V @ W_O).sum(dim=0) + b_O``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t"><span class="str">                The technical derivation of this is as follows. ``v = residual @ W_V[h] +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t"><span class="str">                broadcast_b_V[h]`` for each head ``h`` (where ``b_V`` is broadcast up from shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t"><span class="str">                ``d_head`` to shape ``[position, d_head]``). And ``z = pattern[h] @ v = pattern[h] @</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t"><span class="str">                residual @ W_V[h] + pattern[h] @ broadcast_b_V[h]``. Because ``pattern[h]`` is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t"><span class="str">                ``[destination_position, source_position]`` and ``broadcast_b_V`` is constant along</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t"><span class="str">                the ``(source_)position`` dimension, we're basically just multiplying it by the sum</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t"><span class="str">                of the pattern across the ``source_position`` dimension, which is just ``1``. So it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t"><span class="str">                remains exactly the same, and so is just broadcast across the destination positions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t"><span class="str">            default_prepend_bos: Default behavior of whether to prepend the BOS</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t"><span class="str">                token when the methods of HookedTransformer process input text to tokenize (only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t"><span class="str">                when input is a string).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t"><span class="str">                Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t"><span class="str">                1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t"><span class="str">                2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t"><span class="str">                3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t"><span class="str">                Even for models not explicitly trained with the BOS token, heads often use the first position as a resting position</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t"><span class="str">                and accordingly lose information from the first token, so this empirically seems to give better</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t"><span class="str">                results. Note that you can also locally override the default behavior by passing in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t"><span class="str">                prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t"><span class="str">            from_pretrained_kwargs: Any other optional argument passed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t"><span class="str">                HuggingFace's from_pretrained (e.g. "cache_dir" or "torch_dtype"). Also passed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t"><span class="str">                other HuggingFace functions when compatible. For some models or arguments it doesn't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t"><span class="str">                work, especially for models that are not internally loaded with HuggingFace's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t"><span class="str">                from_pretrained (e.g. SoLU models).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t"><span class="str">            dtype: What data type to load the model in (also sets the dtype of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t"><span class="str">                the HuggingFace model). Set to bfloat16 or float16 if you get out of memory errors when loading</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t"><span class="str">                the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on when tokenizing. Defaults to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t"><span class="str">                "right".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t"><span class="str">            first_n_layers: If specified, only load the first n layers of the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">        <span class="key">if</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"t5"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1229&#x202F;&#x219B;&#x202F;1230</span><span class="annotate long">line 1229 didn't jump to line 1230, because the condition on line 1229 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">            <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">                <span class="str">"Execution stopped: Please use HookedEncoderDecoder to load T5 models instead of HookedTransformer."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">            <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="key">or</span> <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization not supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">        <span class="key">if</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1239&#x202F;&#x219B;&#x202F;1240</span><span class="annotate long">line 1239 didn't jump to line 1240, because the condition on line 1239 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">config</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="nam">qc</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="nam">load_in_4bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="nam">load_in_8bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">            <span class="nam">quant_method</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="nam">load_in_8bit</span><span class="op">,</span> <span class="str">"8-bit quantization is not supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">__version__</span><span class="op">)</span> <span class="op">&lt;</span> <span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="str">"2.1.1"</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for torch versions >= 2.1.1"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="str">"llama"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for Llama models"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">            <span class="key">if</span> <span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">                    <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span> <span class="op">==</span> <span class="str">"bitsandbytes"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Only bitsandbytes quantization is supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">dtype</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="com"># Convert from string to a torch dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">DTYPE_FROM_STRING</span><span class="op">[</span><span class="nam">dtype</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">        <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">from_pretrained_kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1262&#x202F;&#x219B;&#x202F;1265</span><span class="annotate long">line 1262 didn't jump to line 1265, because the condition on line 1262 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="com"># For backwards compatibility with the previous way to do low precision loading</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="com"># This should maybe check the user did not explicitly set dtype *and* torch_dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">from_pretrained_kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1267&#x202F;&#x219B;&#x202F;1271</span><span class="annotate long">line 1267 didn't jump to line 1271, because the condition on line 1267 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">            <span class="op">(</span><span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"torch_dtype"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">            <span class="key">or</span> <span class="nam">dtype</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">        <span class="op">)</span> <span class="key">and</span> <span class="nam">device</span> <span class="key">in</span> <span class="op">[</span><span class="str">"cpu"</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"float16 models may not work on CPU. Consider using a GPU or bfloat16."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">        <span class="com"># Get the model name used in HuggingFace, rather than the alias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">        <span class="com"># Load the config into an HookedTransformerConfig object. If loading from a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">        <span class="com"># checkpoint, the config object will contain the information about the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">        <span class="com"># checkpoint</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">        <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="nam">hf_cfg</span><span class="op">=</span><span class="nam">hf_cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="nam">checkpoint_index</span><span class="op">=</span><span class="nam">checkpoint_index</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="nam">checkpoint_value</span><span class="op">=</span><span class="nam">checkpoint_value</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="nam">n_devices</span><span class="op">=</span><span class="nam">n_devices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">            <span class="nam">first_n_layers</span><span class="op">=</span><span class="nam">first_n_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">                    <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">                    <span class="str">"ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">                <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">                    <span class="str">"You tried to specify center_unembed=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">                    <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">                <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">                    <span class="str">"You tried to specify center_writing_weights=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">                    <span class="str">"Setting center_writing_weights=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">                <span class="nam">center_writing_weights</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span> <span class="key">and</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1312&#x202F;&#x219B;&#x202F;1313</span><span class="annotate long">line 1312 didn't jump to line 1313, because the condition on line 1312 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">                <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">        <span class="com"># Get the state dict of the model (ie a mapping of parameter names to tensors), processed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">        <span class="com"># match the HookedTransformer parameter names.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">,</span> <span class="nam">hf_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span> <span class="op">**</span><span class="nam">from_pretrained_kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">        <span class="com"># Create the HookedTransformer object</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">        <span class="nam">model</span> <span class="op">=</span> <span class="nam">cls</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">            <span class="nam">cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="nam">move_to_device</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">        <span class="nam">model</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1342&#x202F;&#x219B;&#x202F;1345</span><span class="annotate long">line 1342 didn't jump to line 1345, because the condition on line 1342 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">            <span class="nam">model</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">        <span class="nam">print</span><span class="op">(</span><span class="str">f"Loaded pretrained model {model_name} into HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">        <span class="key">return</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained_no_processing</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">        <span class="nam">cls</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">        <span class="str">"""Wrapper for from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t"><span class="str">        Wrapper for from_pretrained with all boolean flags related to simplifying the model set to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t"><span class="str">        False. Refer to from_pretrained for details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t">        <span class="key">return</span> <span class="nam">cls</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">            <span class="nam">model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">    <span class="key">def</span> <span class="nam">init_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">        <span class="str">"""Initialize weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t"><span class="str">        LayerNorm weights are already initialized to 1.0, and all biases are initialized to 0.0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t"><span class="str">        (including LayerNorm), so this just initializes weight matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t"><span class="str">        Weight matrices are set to empty by default (to save space + compute, since they're the bulk</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t"><span class="str">        of the parameters), so it is important to call this if you are not loading in pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t"><span class="str">        weights! Note that this function assumes that weight names being with `W_`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t"><span class="str">        Set seed here to ensure determinism.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t"><span class="str">        This does NOT follow the PyTorch scheme, which as far as I can tell is super out of date but</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t"><span class="str">        no one has gotten round to updating it? https://github.com/pytorch/pytorch/issues/18182</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t"><span class="str">        The default PyTorch scheme is the following: all linear layers use uniform(-1/sqrt(fan_in),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t"><span class="str">        1/sqrt(fan_in)) for weights, and uniform(-1/sqrt(fan_in), 1/sqrt(fan_in)) for biases. For</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t"><span class="str">        biases, fan_in is computed using the fan_in for the weight matrix of the linear layer. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t"><span class="str">        tha it *does not actually* use Kaiming initialization, despite the fact that it calls the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t"><span class="str">        function.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t"><span class="str">        However, for Transformer blocks, it instead initializes biases to zero and weights using Xavier uniform, that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t"><span class="str">        is: uniform(-sqrt(6 / (fan_in + fan_out)), sqrt(6 / (fan_in + fan_out))) for weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t"><span class="str">        PyTorch Transformers are especially bad - TransformerEncoder initializes all layers to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t"><span class="str">        exact same weights?! https://github.com/pytorch/pytorch/issues/72253.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t"><span class="str">        The best paper I've found on transformer initialization is the muP paper, but haven't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t"><span class="str">        integrated those ideas yet: https://arxiv.org/abs/2203.03466</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t"><span class="str">        We split off the initialization into separate functions because muP initialization handles</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t"><span class="str">        different parts of the model differently.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1415&#x202F;&#x219B;&#x202F;1416</span><span class="annotate long">line 1415 didn't jump to line 1416, because the condition on line 1415 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">manual_seed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1418&#x202F;&#x219B;&#x202F;1420</span><span class="annotate long">line 1418 didn't jump to line 1420, because the condition on line 1418 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"muP"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>  <span class="com"># muP uses normal initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">        <span class="str">"""Initialize weights with GPT-2 initialization. Biases are initialized to 0.0 and weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t"><span class="str">        are initialized to N(0, 0.64/d_model) if initializer_range is not set, otherwise std is initializer_range.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">                <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t"><span class="str">        Initialize weights with Xavier initialization -- that is, scale the weights by sqrt(6 /</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t"><span class="str">        (fan_in + fan_out)) for a [-1, 1] uniform distribution, or sqrt(2 / (fan_in + fan_out)) for a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t"><span class="str">        standard normal.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t"><span class="str">        Note that since TransformerLens implements the matrices in the opposite orientation to what</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t"><span class="str">        torch does (e.g. it's d_in x d_out, not d_out x d_in as in torch), we need to calculate it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t"><span class="str">        ourselves.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">                    <span class="nam">init_xavier_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">                    <span class="nam">init_xavier_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t"><span class="str">        Initialize weights with Kaiming initialization -- that is, scale the weights by</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t"><span class="str">        c / sqrt(fan_in), where c = sqrt(2) if the params were immediately preceded by a relu and 1 for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t"><span class="str">        everything else.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t"><span class="str">        Note that the numbers are actually incorrect here when you're using a nonlinearity other</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t"><span class="str">        than relu, e.g. the correct c for SiLu is ~1.74, for tanh it's 5/3 ~= 1.67, and for GeLU it's ~1.57.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t"><span class="str">        But this is unlikely to matter in practice.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t"><span class="str">        I'm just using fan_mode = "fan_in" for now, but it should be trivial to add fan_out.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t"><span class="str">        Again, we have to implement it ourselves because of the orientation of the matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">                    <span class="nam">init_kaiming_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">                    <span class="nam">init_kaiming_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t"><span class="str">        Initialize weights with muParameterization. This involves scaling output weights by a factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t"><span class="str">        of 1/fan_in, input weights and biases by 1, everything else by a factor of 1/sqrt(fan_in).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t"><span class="str">        Also, you need to use muAdamW, which rescales the learning rate for output weights and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t"><span class="str">        hidden weights by a factor of 1/fan_in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t"><span class="str">        All biases are still assumed to be initialized to 0.0, so we only need to change the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t"><span class="str">        weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">                <span class="nam">fan_in</span><span class="op">,</span> <span class="nam">_</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">calc_fan_in_and_fan_out</span><span class="op">(</span><span class="nam">param</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">                <span class="key">if</span> <span class="str">"embed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="nam">float</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">                <span class="key">elif</span> <span class="str">"unembed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">*=</span> <span class="num">3</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="op">-</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">        <span class="str">"""Load &amp; Process State Dict.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t"><span class="str">        Load a state dict into the model, and to apply processing to simplify it. The state dict is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t"><span class="str">        assumed to be in the HookedTransformer format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t"><span class="str">        See the relevant method (same name as the flag) for more details on the folding, centering</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="str">        and processing flags.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="str">            state_dict (dict): The state dict of the model, in HookedTransformer format. fold_ln</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t"><span class="str">            fold_ln (bool, optional): Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t"><span class="str">            center_writing_weights (bool, optional): Whether to center weights writing to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="str">                residual stream (ie set mean to be zero). Due to LayerNorm this doesn't change the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="str">                computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="str">            center_unembed (bool, optional): Whether to center W_U (ie set mean to be zero).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="str">                Softmax is translation invariant so this doesn't affect log probs or loss, but does</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="str">                change logits. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="str">            fold_value_biases (bool, optional): Whether to fold the value biases into the output</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="str">                bias. Because attention patterns add up to 1, the value biases always have a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="str">                constant effect on a layer's output, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t"><span class="str">                associated with. We can factor this all into a single output bias to the layer, and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">                make it easier to interpret the head's output.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices (bool, optional): Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">            model_name (str, optional): checks the model name for special cases of state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t"><span class="str">                loading. Only used for Redwood 2L model currently.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span> <span class="key">and</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1543&#x202F;&#x219B;&#x202F;1544</span><span class="annotate long">line 1543 didn't jump to line 1544, because the condition on line 1543 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">                <span class="str">"With reduced precision, it is advised to use `from_pretrained_no_processing` instead of `from_pretrained`."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1548&#x202F;&#x219B;&#x202F;1553</span><span class="annotate long">line 1548 didn't jump to line 1553</span></span></p>
+    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">                <span class="str">"When running MoE models, it is advised to use a higher precision data type. See docs for more info."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1559&#x202F;&#x219B;&#x202F;1560</span><span class="annotate long">line 1559 didn't jump to line 1560, because the condition on line 1559 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">                    <span class="str">"You are using MoE, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1563&#x202F;&#x219B;&#x202F;1565</span><span class="annotate long">line 1563 didn't jump to line 1565, because the condition on line 1563 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">                    <span class="str">"You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1575&#x202F;&#x219B;&#x202F;1576</span><span class="annotate long">line 1575 didn't jump to line 1576, because the condition on line 1575 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">                    <span class="str">"You are not using LayerNorm, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">                    <span class="str">"This model is using final RMS normalization, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_unembed</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_value_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">        <span class="key">if</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1593&#x202F;&#x219B;&#x202F;1596</span><span class="annotate long">line 1593 didn't jump to line 1596, because the condition on line 1593 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">            <span class="com"># with quantization, parameters should be assigned</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">            <span class="com"># so that quantization settings are not lost</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">assign</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">            <span class="nam">state_dict_keys</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">state_dict_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">}</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">    <span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">        <span class="key">return</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="str">"""Fold Layer Norm. Can also be used to fold RMS Norm, when fold_biases and center_weights are set to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t"><span class="str">        Takes in a state dict from a pretrained model, formatted to be consistent with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t"><span class="str">        HookedTransformer but with LayerNorm weights and biases. Folds these into the neighbouring</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t"><span class="str">        weights. See further_comments.md for more details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t"><span class="str">            state_dict (Dict[str, torch.Tensor]): State dict of pretrained model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t"><span class="str">            fold_biases (bool): Enables folding of LN biases. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t"><span class="str">            center_weights (bool): Enables the centering of weights after folding in LN. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">        <span class="com"># Models that use Grouped Query Attention (Only Mistral at the time of writing) prefix their K/V weights and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="com"># biases with an underscore in order to distinguish them, but folding the LN into them still works the same,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">        <span class="com"># so we just add the underscore if GQA is used (i.e. if `cfg.n_key_value_heads is specified`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">        <span class="nam">gqa</span> <span class="op">=</span> <span class="str">""</span> <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span> <span class="key">else</span> <span class="str">"_"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">            <span class="com"># Fold ln1 into attention - it's important to fold biases first, since biases depend on</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">            <span class="com"># weights but not vice versa The various indexing is just to broadcast ln.b and ln.w</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">            <span class="com"># along every axis other than d_model. Each weight matrix right multiplies. To fold in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">            <span class="com"># the bias, we use the W_ matrix to map it to the hidden space of the layer, so we need</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">            <span class="com"># to sum along axis -2, which is the residual stream space axis.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1632&#x202F;&#x219B;&#x202F;1655</span><span class="annotate long">line 1632 didn't jump to line 1655</span></span></p>
+    <p class="run"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_K"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_V"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">            <span class="com"># Finally, we center the weights reading from the residual stream. The output of the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">            <span class="com"># first part of the LayerNorm is mean 0 and standard deviation 1, so the mean of any</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">            <span class="com"># input vector of the matrix doesn't matter and can be set to zero. Equivalently, the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">            <span class="com"># output of LayerNormPre is orthogonal to the vector of all 1s (because dotting with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">            <span class="com"># that gets the sum), so we can remove the component of the matrix parallel to this.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1673&#x202F;&#x219B;&#x202F;1691</span><span class="annotate long">line 1673 didn't jump to line 1691, because the condition on line 1673 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">            <span class="com"># Fold ln2 into MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1692&#x202F;&#x219B;&#x202F;1699</span><span class="annotate long">line 1692 didn't jump to line 1699</span></span></p>
+    <p class="run"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1703&#x202F;&#x219B;&#x202F;1704</span><span class="annotate long">line 1703 didn't jump to line 1704</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1711&#x202F;&#x219B;&#x202F;1719</span><span class="annotate long">line 1711 didn't jump to line 1719, because the condition on line 1711 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">                    <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">                        <span class="str">"d_model d_mlp -> 1 d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">                        <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"solu"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">                    <span class="com"># Fold ln3 into activation</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">                    <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1721&#x202F;&#x219B;&#x202F;1733</span><span class="annotate long">line 1721 didn't jump to line 1733</span></span></p>
+    <p class="run"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">                            <span class="str">f"blocks.{l}.mlp.b_out"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">                        <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">                            <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">                        <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">                            <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">                        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">                    <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1738&#x202F;&#x219B;&#x202F;1746</span><span class="annotate long">line 1738 didn't jump to line 1746, because the condition on line 1738 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">                        <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">                            <span class="str">"d_mlp d_model -> 1 d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">                            <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">        <span class="com"># Fold ln_final into Unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span> <span class="key">and</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">            <span class="com"># Dumb bug from my old SoLU training code, some models have RMSNorm instead of LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">            <span class="com"># pre unembed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">            <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1760&#x202F;&#x219B;&#x202F;1766</span><span class="annotate long">line 1760 didn't jump to line 1766, because the condition on line 1760 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">            <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span><span class="op">,</span> <span class="str">"d_model d_vocab -> 1 d_vocab"</span><span class="op">,</span> <span class="str">"mean"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">        <span class="str">"""Center Writing Weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t"><span class="str">        Centers the weights of the model that write to the residual stream - W_out, W_E, W_pos and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t"><span class="str">        W_out. This is done by subtracting the mean of the weights from the weights themselves. This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t"><span class="str">        is done in-place. See fold_layer_norm for more details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="str">"pos_embed.W_pos"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                <span class="str">f"blocks.{l}.attn.W_O"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">                <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">            <span class="op">)</span>  <span class="com"># W_O is [head_index, d_model, d_head]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">            <span class="op">)</span>  <span class="com"># b_O is [d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">                    <span class="str">f"blocks.{l}.mlp.W_out"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">                <span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">        <span class="str">"""Center the unembedding weights W_U.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t"><span class="str">        This is done by subtracting the mean of the weights from the weights themselves. This is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t"><span class="str">        done in-place. As softmax is translation invariant, this changes the logits but not the log</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t"><span class="str">        probs, and makes the model logits (slightly) more interpretable - when trying to understand</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t"><span class="str">        how components contribute to the logits, we'll be less misled by components that just add</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t"><span class="str">        something to every logit.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">        <span class="str">"""Fold the value biases into the output bias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t"><span class="str">        Because attention patterns add up to 1, the value biases always have a constant effect on a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t"><span class="str">        head's output. Further, as the outputs of each head in a layer add together, each head's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t"><span class="str">        value bias has a constant effect on the *layer's* output, which can make it harder to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t"><span class="str">        interpret the effect of any given head, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t"><span class="str">        associated with. We can factor this all into a single output bias to the layer, and make it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t"><span class="str">        easier to interpret the head's output. Formally, we take b_O_new = b_O_original +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t"><span class="str">        sum_head(b_V_head @ W_O_head).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">            <span class="com"># shape [head_index, d_head]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1828&#x202F;&#x219B;&#x202F;1831</span><span class="annotate long">line 1828 didn't jump to line 1831, because the condition on line 1828 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">                    <span class="nam">b_V</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">,</span> <span class="nam">repeats</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span> <span class="op">//</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">            <span class="com"># [head_index, d_head, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">            <span class="com"># [d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">            <span class="nam">b_O_original</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">folded_b_O</span> <span class="op">=</span> <span class="nam">b_O_original</span> <span class="op">+</span> <span class="op">(</span><span class="nam">b_V</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_O</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">folded_b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1842&#x202F;&#x219B;&#x202F;1845</span><span class="annotate long">line 1842 didn't jump to line 1845, because the condition on line 1842 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">    <span class="key">def</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">        <span class="str">"""Experimental method for managing queries, keys and values.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t"><span class="str">        As argued in [A Mathematical Framework for Transformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t"><span class="str">        Circuits](https://transformer-circuits.pub/2021/framework/index.html), queries, keys and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t"><span class="str">        values are somewhat arbitrary intermediate terms when computing with the low rank factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t"><span class="str">        matrices W_QK = W_Q @ W_K.T and W_OV = W_V @ W_O, and these matrices are the only thing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t"><span class="str">        determining head behaviour. But there are many ways to find a low rank factorization to a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t"><span class="str">        given matrix, and hopefully some of these are more interpretable than others! This method is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t"><span class="str">        one attempt, which makes all of the matrices have orthogonal rows or columns, W_O into a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t"><span class="str">        rotation and W_Q and W_K having the nth column in each having the same norm. The formula is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t"><span class="str">        $W_V = U @ S,W_O=Vh.T,W_Q=U@S.sqrt(),W_K=Vh@S.sqrt()$.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t"><span class="str">        More details:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t"><span class="str">        If W_OV = U @ S @ Vh.T in its singular value decomposition, (where S is in R^d_head not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t"><span class="str">        R^d_model, as W_OV is low rank), W_OV = (U @ S) @ (Vh.T) is an equivalent low rank</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t"><span class="str">        factorisation, where rows/columns of each matrix are orthogonal! So setting $W_V=US$ and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t"><span class="str">        $W_O=Vh.T$ works just as well. I *think* this is a more interpretable setup, because now</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t"><span class="str">        $W_O$ is just a rotation, and doesn't change the norm, so $z$ has the same norm as the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t"><span class="str">        result of the head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t"><span class="str">        For $W_QK = W_Q @ W_K.T$ we use the refactor $W_Q = U @ S.sqrt()$ and $W_K = Vh @ S.sqrt()$,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t"><span class="str">        which is also equivalent ($S==S.sqrt() @ S.sqrt()$ as $S$ is diagonal). Here we keep the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t"><span class="str">        matrices as having the same norm, since there's not an obvious asymmetry between the keys</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t"><span class="str">        and queries.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t"><span class="str">        Biases are more fiddly to deal with. For OV it's pretty easy - we just need (x @ W_V + b_V)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t"><span class="str">        @ W_O + b_O to be preserved, so we can set b_V' = 0. and b_O' = b_V @ W_O + b_O (note that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t"><span class="str">        b_V in R^{head_index x d_head} while b_O in R^{d_model}, so we need to sum b_V @ W_O along</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t"><span class="str">        the head_index dimension too).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t"><span class="str">        For QK it's messy - we need to preserve the bilinear form of (x @ W_Q + b_Q) * (y @ W_K +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t"><span class="str">        b_K), which is fairly messy. To deal with the biases, we concatenate them to W_Q and W_K to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t"><span class="str">        simulate a d_model+1 dimensional input (whose final coordinate is always 1), do the SVD</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t"><span class="str">        factorization on this effective matrix, then separate out into final weights and biases.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"You can't refactor the QK circuit when using rotary embeddings (as the QK matrix depends on the position of the query and key)"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">            <span class="com"># W_QK = W_Q @ W_K.T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">            <span class="com"># Concatenate biases to make a d_model+1 input dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">            <span class="nam">W_Q_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">            <span class="nam">W_K_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">            <span class="nam">W_Q_eff_even</span><span class="op">,</span> <span class="nam">W_K_eff_even_T</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">                <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_Q_eff</span><span class="op">,</span> <span class="nam">W_K_eff</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">)</span><span class="op">.</span><span class="nam">make_even</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">pair</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">            <span class="nam">W_K_eff_even</span> <span class="op">=</span> <span class="nam">W_K_eff_even_T</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">            <span class="com"># W_OV = W_V @ W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">            <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t">            <span class="com"># Factors the bias to be consistent.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">            <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">            <span class="nam">b_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">            <span class="com"># Add singleton dimension for broadcasting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">            <span class="nam">b_V_expanded</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">b_V</span><span class="op">,</span> <span class="str">"head_index d_head -> head_index d_head 1"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1931" href="#t1931">1931</a></span><span class="t">            <span class="com"># Element-wise multiplication of b_V and W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1932" href="#t1932">1932</a></span><span class="t">            <span class="nam">b_V_times_W_O</span> <span class="op">=</span> <span class="nam">b_V_expanded</span> <span class="op">*</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1933" href="#t1933">1933</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1934" href="#t1934">1934</a></span><span class="t">            <span class="com"># Helper class to efficiently deal with low rank factored matrices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1935" href="#t1935">1935</a></span><span class="t">            <span class="nam">W_OV</span> <span class="op">=</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1936" href="#t1936">1936</a></span><span class="t">            <span class="nam">U</span><span class="op">,</span> <span class="nam">S</span><span class="op">,</span> <span class="nam">Vh</span> <span class="op">=</span> <span class="nam">W_OV</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1937" href="#t1937">1937</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">U</span> <span class="op">@</span> <span class="nam">S</span><span class="op">.</span><span class="nam">diag_embed</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1938" href="#t1938">1938</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="nam">Vh</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1939" href="#t1939">1939</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1940" href="#t1940">1940</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1941" href="#t1941">1941</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1942" href="#t1942">1942</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_result</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_result</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1943" href="#t1943">1943</a></span><span class="t">        <span class="str">"""Toggle whether to explicitly calculate and expose the result for each attention head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1944" href="#t1944">1944</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1945" href="#t1945">1945</a></span><span class="t"><span class="str">        Useful for interpretability but can easily burn through GPU memory.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1946" href="#t1946">1946</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1947" href="#t1947">1947</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span> <span class="op">=</span> <span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1934" href="#t1934">1934</a></span><span class="t">            <span class="com"># Sum over d_head and head_index dimensions</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1935" href="#t1935">1935</a></span><span class="t">            <span class="nam">b_V_contribution</span> <span class="op">=</span> <span class="nam">b_V_times_W_O</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1936" href="#t1936">1936</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1937" href="#t1937">1937</a></span><span class="t">            <span class="nam">effective_bias</span> <span class="op">=</span> <span class="nam">b_O</span> <span class="op">+</span> <span class="nam">b_V_contribution</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1938" href="#t1938">1938</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1939" href="#t1939">1939</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">effective_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1940" href="#t1940">1940</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1941" href="#t1941">1941</a></span><span class="t">            <span class="com"># Helper class to efficiently deal with low rank factored matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1942" href="#t1942">1942</a></span><span class="t">            <span class="nam">W_OV</span> <span class="op">=</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1943" href="#t1943">1943</a></span><span class="t">            <span class="nam">U</span><span class="op">,</span> <span class="nam">S</span><span class="op">,</span> <span class="nam">Vh</span> <span class="op">=</span> <span class="nam">W_OV</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1944" href="#t1944">1944</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">U</span> <span class="op">@</span> <span class="nam">S</span><span class="op">.</span><span class="nam">diag_embed</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1945" href="#t1945">1945</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="nam">Vh</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1946" href="#t1946">1946</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1947" href="#t1947">1947</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1948" href="#t1948">1948</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1949" href="#t1949">1949</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_split_qkv_input</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_split_qkv_input</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1950" href="#t1950">1950</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1951" href="#t1951">1951</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1952" href="#t1952">1952</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1953" href="#t1953">1953</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="op">=</span> <span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1954" href="#t1954">1954</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1955" href="#t1955">1955</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_hook_mlp_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_hook_mlp_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1956" href="#t1956">1956</a></span><span class="t">        <span class="str">"""Toggles whether to allow storing and editing inputs to each MLP layer."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1957" href="#t1957">1957</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1958" href="#t1958">1958</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">,</span> <span class="str">"Can't use hook_mlp_in with attn_only model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1959" href="#t1959">1959</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span> <span class="op">=</span> <span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1960" href="#t1960">1960</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1961" href="#t1961">1961</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1962" href="#t1962">1962</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1963" href="#t1963">1963</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1964" href="#t1964">1964</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1965" href="#t1965">1965</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1966" href="#t1966">1966</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1967" href="#t1967">1967</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_ungroup_grouped_query_attention</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">ungroup_grouped_query_attention</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1968" href="#t1968">1968</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1969" href="#t1969">1969</a></span><span class="t"><span class="str">        Toggles whether to ungroup the grouped key and value heads in models with grouped query attention (GQA).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1970" href="#t1970">1970</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1971" href="#t1971">1971</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">ungroup_grouped_query_attention</span> <span class="op">=</span> <span class="nam">ungroup_grouped_query_attention</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1987&#x202F;&#x219B;&#x202F;1990</span><span class="annotate long">line 1987 didn't jump to line 1990, because the condition on line 1987 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1991&#x202F;&#x219B;&#x202F;2002</span><span class="annotate long">line 1991 didn't jump to line 2002, because the condition on line 1991 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2000&#x202F;&#x219B;&#x202F;2001</span><span class="annotate long">line 2000 didn't jump to line 2001, because the condition on line 2000 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2091&#x202F;&#x219B;&#x202F;2098</span><span class="annotate long">line 2091 didn't jump to line 2098, because the condition on line 2091 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2100&#x202F;&#x219B;&#x202F;2106</span><span class="annotate long">line 2100 didn't jump to line 2106, because the condition on line 2100 was never false</span></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2101&#x202F;&#x219B;&#x202F;2104</span><span class="annotate long">line 2101 didn't jump to line 2104, because the condition on line 2101 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2110" href="#t2110">2110</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2110&#x202F;&#x219B;&#x202F;2115</span><span class="annotate long">line 2110 didn't jump to line 2115, because the condition on line 2110 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2120&#x202F;&#x219B;&#x202F;2142</span><span class="annotate long">line 2120 didn't jump to line 2142, because the condition on line 2120 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2124&#x202F;&#x219B;&#x202F;2131</span><span class="annotate long">line 2124 didn't jump to line 2131, because the condition on line 2124 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2131&#x202F;&#x219B;&#x202F;2136</span><span class="annotate long">line 2131 didn't jump to line 2136, because the condition on line 2131 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2150" href="#t2150">2150</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2151&#x202F;&#x219B;&#x202F;2172</span><span class="annotate long">line 2151 didn't jump to line 2172, because the condition on line 2151 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2177" href="#t2177">2177</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2180&#x202F;&#x219B;&#x202F;2181</span><span class="annotate long">line 2180 didn't jump to line 2181, because the condition on line 2180 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2187" href="#t2187">2187</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2194&#x202F;&#x219B;&#x202F;2206</span><span class="annotate long">line 2194 didn't jump to line 2206, because the condition on line 2194 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2208&#x202F;&#x219B;&#x202F;2209</span><span class="annotate long">line 2208 didn't jump to line 2209, because the condition on line 2208 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2211&#x202F;&#x219B;&#x202F;2219</span><span class="annotate long">line 2211 didn't jump to line 2219, because the condition on line 2211 was never false</span></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2212&#x202F;&#x219B;&#x202F;2214</span><span class="annotate long">line 2212 didn't jump to line 2214, because the condition on line 2212 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1949" href="#t1949">1949</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_result</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_result</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1950" href="#t1950">1950</a></span><span class="t">        <span class="str">"""Toggle whether to explicitly calculate and expose the result for each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1951" href="#t1951">1951</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1952" href="#t1952">1952</a></span><span class="t"><span class="str">        Useful for interpretability but can easily burn through GPU memory.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1953" href="#t1953">1953</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1954" href="#t1954">1954</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span> <span class="op">=</span> <span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1955" href="#t1955">1955</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1956" href="#t1956">1956</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_split_qkv_input</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_split_qkv_input</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1957" href="#t1957">1957</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1958" href="#t1958">1958</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1959" href="#t1959">1959</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1960" href="#t1960">1960</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="op">=</span> <span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1961" href="#t1961">1961</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1962" href="#t1962">1962</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_hook_mlp_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_hook_mlp_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1963" href="#t1963">1963</a></span><span class="t">        <span class="str">"""Toggles whether to allow storing and editing inputs to each MLP layer."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1964" href="#t1964">1964</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1965" href="#t1965">1965</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">,</span> <span class="str">"Can't use hook_mlp_in with attn_only model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1966" href="#t1966">1966</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span> <span class="op">=</span> <span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1967" href="#t1967">1967</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1968" href="#t1968">1968</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1969" href="#t1969">1969</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1970" href="#t1970">1970</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1971" href="#t1971">1971</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_ungroup_grouped_query_attention</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">ungroup_grouped_query_attention</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t"><span class="str">        Toggles whether to ungroup the grouped key and value heads in models with grouped query attention (GQA).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">ungroup_grouped_query_attention</span> <span class="op">=</span> <span class="nam">ungroup_grouped_query_attention</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1994&#x202F;&#x219B;&#x202F;1997</span><span class="annotate long">line 1994 didn't jump to line 1997, because the condition on line 1994 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1998&#x202F;&#x219B;&#x202F;2009</span><span class="annotate long">line 1998 didn't jump to line 2009, because the condition on line 1998 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2007&#x202F;&#x219B;&#x202F;2008</span><span class="annotate long">line 2007 didn't jump to line 2008, because the condition on line 2007 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2098&#x202F;&#x219B;&#x202F;2105</span><span class="annotate long">line 2098 didn't jump to line 2105, because the condition on line 2098 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2107&#x202F;&#x219B;&#x202F;2113</span><span class="annotate long">line 2107 didn't jump to line 2113, because the condition on line 2107 was never false</span></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2108&#x202F;&#x219B;&#x202F;2111</span><span class="annotate long">line 2108 didn't jump to line 2111, because the condition on line 2108 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2110" href="#t2110">2110</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2117&#x202F;&#x219B;&#x202F;2122</span><span class="annotate long">line 2117 didn't jump to line 2122, because the condition on line 2117 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2127&#x202F;&#x219B;&#x202F;2149</span><span class="annotate long">line 2127 didn't jump to line 2149, because the condition on line 2127 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2131&#x202F;&#x219B;&#x202F;2138</span><span class="annotate long">line 2131 didn't jump to line 2138, because the condition on line 2131 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2138&#x202F;&#x219B;&#x202F;2143</span><span class="annotate long">line 2138 didn't jump to line 2143, because the condition on line 2138 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2150" href="#t2150">2150</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2158&#x202F;&#x219B;&#x202F;2179</span><span class="annotate long">line 2158 didn't jump to line 2179, because the condition on line 2158 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2177" href="#t2177">2177</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2187" href="#t2187">2187</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2187&#x202F;&#x219B;&#x202F;2188</span><span class="annotate long">line 2187 didn't jump to line 2188, because the condition on line 2187 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2201&#x202F;&#x219B;&#x202F;2213</span><span class="annotate long">line 2201 didn't jump to line 2213, because the condition on line 2201 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2215&#x202F;&#x219B;&#x202F;2216</span><span class="annotate long">line 2215 didn't jump to line 2216, because the condition on line 2215 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2217" href="#t2217">2217</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2227" href="#t2227">2227</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2264&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2264&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2264 didn't run the list comprehension on line 2264, 2) line 2264 didn't return from function 'W_K', because the return on line 2264 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2269&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2269&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2269 didn't run the list comprehension on line 2269, 2) line 2269 didn't return from function 'W_Q', because the return on line 2269 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2274&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2274&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2274 didn't run the list comprehension on line 2274, 2) line 2274 didn't return from function 'W_V', because the return on line 2274 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2279&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2279&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2279 didn't run the list comprehension on line 2279, 2) line 2279 didn't return from function 'W_O', because the return on line 2279 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2284&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2284&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2284 didn't run the list comprehension on line 2284, 2) line 2284 didn't return from function 'W_in', because the return on line 2284 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2218&#x202F;&#x219B;&#x202F;2226</span><span class="annotate long">line 2218 didn't jump to line 2226, because the condition on line 2218 was never false</span></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2219&#x202F;&#x219B;&#x202F;2221</span><span class="annotate long">line 2219 didn't jump to line 2221, because the condition on line 2219 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2227" href="#t2227">2227</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2271&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2271&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2271 didn't run the list comprehension on line 2271, 2) line 2271 didn't return from function 'W_K', because the return on line 2271 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2276&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2276&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2276 didn't run the list comprehension on line 2276, 2) line 2276 didn't return from function 'W_Q', because the return on line 2276 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2281&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2281&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2281 didn't run the list comprehension on line 2281, 2) line 2281 didn't return from function 'W_V', because the return on line 2281 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2286&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2286&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2286 didn't run the list comprehension on line 2286, 2) line 2286 didn't return from function 'W_O', because the return on line 2286 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2291&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2291&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2291 didn't run the list comprehension on line 2291, 2) line 2291 didn't return from function 'W_in', because the return on line 2291 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2296" href="#t2296">2296</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2300&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2300&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2300 didn't run the list comprehension on line 2300, 2) line 2300 didn't return from function 'W_out', because the return on line 2300 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2305&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2305&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2305 didn't run the list comprehension on line 2305, 2) line 2305 didn't return from function 'b_K', because the return on line 2305 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2310&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2310&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2310 didn't run the list comprehension on line 2310, 2) line 2310 didn't return from function 'b_Q', because the return on line 2310 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2315" href="#t2315">2315</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2315&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2315&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2315 didn't run the list comprehension on line 2315, 2) line 2315 didn't return from function 'b_V', because the return on line 2315 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2320&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2320&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2320 didn't run the list comprehension on line 2320, 2) line 2320 didn't return from function 'b_O', because the return on line 2320 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2325&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2325&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2325 didn't run the list comprehension on line 2325, 2) line 2325 didn't return from function 'b_in', because the return on line 2325 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2330&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2330&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2330 didn't run the list comprehension on line 2330, 2) line 2330 didn't return from function 'b_out', because the return on line 2330 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2347" href="#t2347">2347</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2355" href="#t2355">2355</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2368&#x202F;&#x219B;&#x202F;2369</span><span class="annotate long">line 2368 didn't jump to line 2369, because the condition on line 2368 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2381" href="#t2381">2381</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2307&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2307&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2307 didn't run the list comprehension on line 2307, 2) line 2307 didn't return from function 'W_out', because the return on line 2307 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2312&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2312&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2312 didn't run the list comprehension on line 2312, 2) line 2312 didn't return from function 'b_K', because the return on line 2312 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2315" href="#t2315">2315</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2317&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2317&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2317 didn't run the list comprehension on line 2317, 2) line 2317 didn't return from function 'b_Q', because the return on line 2317 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2322&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2322&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2322 didn't run the list comprehension on line 2322, 2) line 2322 didn't return from function 'b_V', because the return on line 2322 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2327&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2327&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2327 didn't run the list comprehension on line 2327, 2) line 2327 didn't return from function 'b_O', because the return on line 2327 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2332&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2332&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2332 didn't run the list comprehension on line 2332, 2) line 2332 didn't return from function 'b_in', because the return on line 2332 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2337&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2337&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2337 didn't run the list comprehension on line 2337, 2) line 2337 didn't return from function 'b_out', because the return on line 2337 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2347" href="#t2347">2347</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2355" href="#t2355">2355</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2375&#x202F;&#x219B;&#x202F;2376</span><span class="annotate long">line 2375 didn't jump to line 2376, because the condition on line 2375 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2381" href="#t2381">2381</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2384" href="#t2384">2384</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2387" href="#t2387">2387</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2387" href="#t2387">2387</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2414" href="#t2414">2414</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2421" href="#t2421">2421</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2424" href="#t2424">2424</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2424" href="#t2424">2424</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2430" href="#t2430">2430</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2461" href="#t2461">2461</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2462" href="#t2462">2462</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2463" href="#t2463">2463</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2464" href="#t2464">2464</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2465" href="#t2465">2465</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2466" href="#t2466">2466</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2461" href="#t2461">2461</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2462" href="#t2462">2462</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2463" href="#t2463">2463</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2464" href="#t2464">2464</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2465" href="#t2465">2465</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2466" href="#t2466">2466</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2467" href="#t2467">2467</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2468" href="#t2468">2468</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2469" href="#t2469">2469</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2470" href="#t2470">2470</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2471" href="#t2471">2471</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2472" href="#t2472">2472</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2473" href="#t2473">2473</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2474" href="#t2474">2474</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2475" href="#t2475">2475</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2476" href="#t2476">2476</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2477" href="#t2477">2477</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2478" href="#t2478">2478</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2479" href="#t2479">2479</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2480" href="#t2480">2480</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2481" href="#t2481">2481</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2482" href="#t2482">2482</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2483" href="#t2483">2483</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2484" href="#t2484">2484</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2485" href="#t2485">2485</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2486" href="#t2486">2486</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2487" href="#t2487">2487</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2488" href="#t2488">2488</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2489" href="#t2489">2489</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2490" href="#t2490">2490</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2491" href="#t2491">2491</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2492" href="#t2492">2492</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2493" href="#t2493">2493</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2468" href="#t2468">2468</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2469" href="#t2469">2469</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2470" href="#t2470">2470</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2471" href="#t2471">2471</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2472" href="#t2472">2472</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2473" href="#t2473">2473</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2474" href="#t2474">2474</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2475" href="#t2475">2475</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2476" href="#t2476">2476</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2477" href="#t2477">2477</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2478" href="#t2478">2478</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2479" href="#t2479">2479</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2480" href="#t2480">2480</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2481" href="#t2481">2481</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2482" href="#t2482">2482</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2483" href="#t2483">2483</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2484" href="#t2484">2484</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2485" href="#t2485">2485</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2486" href="#t2486">2486</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2487" href="#t2487">2487</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2488" href="#t2488">2488</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2489" href="#t2489">2489</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2490" href="#t2490">2490</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2491" href="#t2491">2491</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2492" href="#t2492">2492</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2493" href="#t2493">2493</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2494" href="#t2494">2494</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2495" href="#t2495">2495</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2496" href="#t2496">2496</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2497" href="#t2497">2497</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2498" href="#t2498">2498</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2499" href="#t2499">2499</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2500" href="#t2500">2500</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -2584,7 +2591,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
index 93d1eddfd..e6875a453 100644
--- a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
+++ b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">55 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">54<span class="text"> run</span></button>
+            <span class="text">54 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">53<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">1<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">1<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -90,160 +90,159 @@ <h2>
     <p class="pln"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">from</span> <span class="nam">typing</span> <span class="key">import</span> <span class="nam">Optional</span><span class="op">,</span> <span class="nam">Union</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">import</span> <span class="nam">fancy_einsum</span> <span class="key">as</span> <span class="nam">einsum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">from</span> <span class="nam">typeguard</span> <span class="key">import</span> <span class="nam">typechecked</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">FactoredMatrix</span> <span class="key">import</span> <span class="nam">FactoredMatrix</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">HookedTransformer</span> <span class="key">import</span> <span class="nam">HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t"><span class="nam">OUTPUT_EMBEDDING</span> <span class="op">=</span> <span class="str">"unembed.W_U"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t"><span class="nam">VECTOR_TYPES</span> <span class="op">=</span> <span class="op">[</span><span class="str">"OV"</span><span class="op">,</span> <span class="str">"w_in"</span><span class="op">,</span> <span class="str">"w_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t"><span class="key">from</span> <span class="nam">typeguard</span> <span class="key">import</span> <span class="nam">typechecked</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">from</span> <span class="nam">typing_extensions</span> <span class="key">import</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">FactoredMatrix</span> <span class="key">import</span> <span class="nam">FactoredMatrix</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">HookedTransformer</span> <span class="key">import</span> <span class="nam">HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t"><span class="nam">OUTPUT_EMBEDDING</span> <span class="op">=</span> <span class="str">"unembed.W_U"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t"><span class="nam">VECTOR_TYPES</span> <span class="op">=</span> <span class="op">[</span><span class="str">"OV"</span><span class="op">,</span> <span class="str">"w_in"</span><span class="op">,</span> <span class="str">"w_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t"><span class="key">class</span> <span class="nam">SVDInterpreter</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">model</span><span class="op">:</span> <span class="nam">HookedTransformer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">model</span> <span class="op">=</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">params</span> <span class="op">=</span> <span class="op">{</span><span class="nam">name</span><span class="op">:</span> <span class="nam">param</span> <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">model</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">    <span class="op">@</span><span class="nam">typechecked</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_singular_vectors</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">        <span class="nam">vector_type</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"OV"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"w_in"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"w_out"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">        <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">        <span class="nam">num_vectors</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">        <span class="nam">head_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">        <span class="str">"""Gets the singular vectors for a given vector type, layer, and optionally head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str">        This tensor can then be plotted using Neel's PySvelte, as demonstrated in the demo for this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str">        feature. The demo also points out some "gotchas" in this feature - numerical instability</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str">        means inconsistency across devices, and the default HookedTransformer parameters don't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str">        replicate the original SVD post very well. So I'd recommend checking out the demo if you</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="str">        want to use this!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"><span class="str">        .. code-block:: python</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t"><span class="str">            from transformer_lens import HookedTransformer, SVDInterpreter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t"><span class="str">            model = HookedTransformer.from_pretrained('gpt2-medium')</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"><span class="str">            svd_interpreter = SVDInterpreter(model)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="str">            ov = svd_interpreter.get_singular_vectors('OV', layer_index=22, head_index=10)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="str">            all_tokens = [model.to_str_tokens(np.array([i])) for i in range(model.cfg.d_vocab)]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="str">            all_tokens = [all_tokens[i][0] for i in range(model.cfg.d_vocab)]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str">            def plot_matrix(matrix, tokens, k=10, filter="topk"):</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="str">                pysvelte.TopKTable(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="str">                    tokens=all_tokens,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="str">                    activations=matrix,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="str">                    obj_type="SVD direction",</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"><span class="str">                    k=k,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"><span class="str">                    filter=filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t"><span class="str">                ).show()</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"><span class="str">            plot_matrix(ov, all_tokens)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="str">            vector_type: Type of the vector:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str">                - "OV": Singular vectors of the OV matrix for a particular layer and head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="str">                - "w_in": Singular vectors of the w_in matrix for a particular layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"><span class="str">                - "w_out": Singular vectors of the w_out matrix for a particular layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"><span class="str">            layer_index: The index of the layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"><span class="str">            num_vectors: Number of vectors.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"><span class="str">            head_index: Index of the head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t">        <span class="key">if</span> <span class="nam">head_index</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t">            <span class="key">assert</span> <span class="nam">vector_type</span> <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t">                <span class="str">"w_in"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t">                <span class="str">"w_out"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t">            <span class="op">]</span><span class="op">,</span> <span class="str">f"Head index optional only for w_in and w_out, got {vector_type}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t">        <span class="nam">matrix</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">FactoredMatrix</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t">        <span class="key">if</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"OV"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t">            <span class="key">assert</span> <span class="nam">head_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_OV_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">,</span> <span class="nam">head_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t">            <span class="nam">V</span> <span class="op">=</span> <span class="nam">matrix</span><span class="op">.</span><span class="nam">Vh</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t">        <span class="key">elif</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"w_in"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_w_in_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t">            <span class="nam">_</span><span class="op">,</span> <span class="nam">_</span><span class="op">,</span> <span class="nam">V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">linalg</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="nam">matrix</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t">        <span class="key">elif</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"w_out"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">94&#x202F;&#x219B;&#x202F;99</span><span class="annotate long">line 94 didn't jump to line 99, because the condition on line 94 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_w_out_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t">            <span class="nam">_</span><span class="op">,</span> <span class="nam">_</span><span class="op">,</span> <span class="nam">V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">linalg</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="nam">matrix</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Vector type must be in {VECTOR_TYPES}, instead got {vector_type}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_singular_vectors_from_matrix</span><span class="op">(</span><span class="nam">V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="nam">OUTPUT_EMBEDDING</span><span class="op">]</span><span class="op">,</span> <span class="nam">num_vectors</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_singular_vectors_from_matrix</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t">        <span class="nam">V</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">FactoredMatrix</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t">        <span class="nam">embedding</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t">        <span class="nam">num_vectors</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t">        <span class="str">"""Returns the top num_vectors singular vectors from a matrix."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t">        <span class="nam">vectors_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">num_vectors</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">            <span class="nam">activations</span> <span class="op">=</span> <span class="nam">V</span><span class="op">[</span><span class="nam">i</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">.</span><span class="nam">float</span><span class="op">(</span><span class="op">)</span> <span class="op">@</span> <span class="nam">embedding</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t">            <span class="nam">vectors_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">activations</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">        <span class="nam">vectors</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">vectors_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">        <span class="key">assert</span> <span class="nam">vectors</span><span class="op">.</span><span class="nam">shape</span> <span class="op">==</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t">            <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t">            <span class="nam">num_vectors</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Vectors shape should be {self.cfg.d_vocab, 1, num_vectors} but got {vectors.shape}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">        <span class="key">return</span> <span class="nam">vectors</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_OV_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">head_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">FactoredMatrix</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t">        <span class="str">"""Gets the OV matrix for a particular layer and head."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">head_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Head index must be between 0 and {self.cfg.n_heads-1} but got {head_index}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">        <span class="nam">W_V</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">        <span class="nam">W_O</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">        <span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">W_V</span><span class="op">[</span><span class="nam">head_index</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">[</span><span class="nam">head_index</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_w_in_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">        <span class="str">"""Gets the w_in matrix for a particular layer."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t">        <span class="nam">w_in</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.mlp.W_in"</span><span class="op">]</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">        <span class="key">if</span> <span class="str">f"blocks.{layer_index}.ln2.w"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">:</span>  <span class="com"># If fold_ln == False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">            <span class="nam">ln_2</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">            <span class="key">return</span> <span class="nam">einsum</span><span class="op">.</span><span class="nam">einsum</span><span class="op">(</span><span class="str">"out in, in -> out in"</span><span class="op">,</span> <span class="nam">w_in</span><span class="op">,</span> <span class="nam">ln_2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">        <span class="key">return</span> <span class="nam">w_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_w_out_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">        <span class="str">"""Gets the w_out matrix for a particular layer."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t"><span class="key">class</span> <span class="nam">SVDInterpreter</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t">    <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">model</span><span class="op">:</span> <span class="nam">HookedTransformer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">model</span> <span class="op">=</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">params</span> <span class="op">=</span> <span class="op">{</span><span class="nam">name</span><span class="op">:</span> <span class="nam">param</span> <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">model</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t">    <span class="op">@</span><span class="nam">typechecked</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_singular_vectors</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">        <span class="nam">vector_type</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"OV"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"w_in"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"w_out"</span><span class="op">]</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">        <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">        <span class="nam">num_vectors</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">        <span class="nam">head_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">        <span class="str">"""Gets the singular vectors for a given vector type, layer, and optionally head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t"><span class="str">        This tensor can then be plotted using Neel's PySvelte, as demonstrated in the demo for this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str">        feature. The demo also points out some "gotchas" in this feature - numerical instability</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str">        means inconsistency across devices, and the default HookedTransformer parameters don't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str">        replicate the original SVD post very well. So I'd recommend checking out the demo if you</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str">        want to use this!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t"><span class="str">        .. code-block:: python</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"><span class="str">            from transformer_lens import HookedTransformer, SVDInterpreter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"><span class="str">            model = HookedTransformer.from_pretrained('gpt2-medium')</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t"><span class="str">            svd_interpreter = SVDInterpreter(model)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"><span class="str">            ov = svd_interpreter.get_singular_vectors('OV', layer_index=22, head_index=10)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"><span class="str">            all_tokens = [model.to_str_tokens(np.array([i])) for i in range(model.cfg.d_vocab)]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="str">            all_tokens = [all_tokens[i][0] for i in range(model.cfg.d_vocab)]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"><span class="str">            def plot_matrix(matrix, tokens, k=10, filter="topk"):</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str">                pysvelte.TopKTable(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="str">                    tokens=all_tokens,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="str">                    activations=matrix,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="str">                    obj_type="SVD direction",</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="str">                    k=k,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"><span class="str">                    filter=filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"><span class="str">                ).show()</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t"><span class="str">            plot_matrix(ov, all_tokens)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="str">            vector_type: Type of the vector:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="str">                - "OV": Singular vectors of the OV matrix for a particular layer and head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str">                - "w_in": Singular vectors of the w_in matrix for a particular layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="str">                - "w_out": Singular vectors of the w_out matrix for a particular layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"><span class="str">            layer_index: The index of the layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"><span class="str">            num_vectors: Number of vectors.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"><span class="str">            head_index: Index of the head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">        <span class="key">if</span> <span class="nam">head_index</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t">            <span class="key">assert</span> <span class="nam">vector_type</span> <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t">                <span class="str">"w_in"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t">                <span class="str">"w_out"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t">            <span class="op">]</span><span class="op">,</span> <span class="str">f"Head index optional only for w_in and w_out, got {vector_type}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t">        <span class="nam">matrix</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">FactoredMatrix</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t">        <span class="key">if</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"OV"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t">            <span class="key">assert</span> <span class="nam">head_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_OV_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">,</span> <span class="nam">head_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t">            <span class="nam">V</span> <span class="op">=</span> <span class="nam">matrix</span><span class="op">.</span><span class="nam">Vh</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t">        <span class="key">elif</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"w_in"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_w_in_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t">            <span class="nam">_</span><span class="op">,</span> <span class="nam">_</span><span class="op">,</span> <span class="nam">V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">linalg</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="nam">matrix</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t">        <span class="key">elif</span> <span class="nam">vector_type</span> <span class="op">==</span> <span class="str">"w_out"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">93&#x202F;&#x219B;&#x202F;98</span><span class="annotate long">line 93 didn't jump to line 98, because the condition on line 93 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t">            <span class="nam">matrix</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_w_out_matrix</span><span class="op">(</span><span class="nam">layer_index</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t">            <span class="nam">_</span><span class="op">,</span> <span class="nam">_</span><span class="op">,</span> <span class="nam">V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">linalg</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="nam">matrix</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Vector type must be in {VECTOR_TYPES}, instead got {vector_type}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_get_singular_vectors_from_matrix</span><span class="op">(</span><span class="nam">V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="nam">OUTPUT_EMBEDDING</span><span class="op">]</span><span class="op">,</span> <span class="nam">num_vectors</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_singular_vectors_from_matrix</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t">        <span class="nam">V</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">FactoredMatrix</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t">        <span class="nam">embedding</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t">        <span class="nam">num_vectors</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t">        <span class="str">"""Returns the top num_vectors singular vectors from a matrix."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t">        <span class="nam">vectors_list</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">num_vectors</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t">            <span class="nam">activations</span> <span class="op">=</span> <span class="nam">V</span><span class="op">[</span><span class="nam">i</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">.</span><span class="nam">float</span><span class="op">(</span><span class="op">)</span> <span class="op">@</span> <span class="nam">embedding</span>  <span class="com"># type: ignore</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">            <span class="nam">vectors_list</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">activations</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">        <span class="nam">vectors</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="nam">vectors_list</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t">        <span class="key">assert</span> <span class="nam">vectors</span><span class="op">.</span><span class="nam">shape</span> <span class="op">==</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_vocab</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">            <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t">            <span class="nam">num_vectors</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Vectors shape should be {self.cfg.d_vocab, 1, num_vectors} but got {vectors.shape}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t">        <span class="key">return</span> <span class="nam">vectors</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_OV_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">head_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">FactoredMatrix</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t">        <span class="str">"""Gets the OV matrix for a particular layer and head."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">head_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Head index must be between 0 and {self.cfg.n_heads-1} but got {head_index}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t">        <span class="nam">W_V</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">        <span class="nam">W_O</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">        <span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">W_V</span><span class="op">[</span><span class="nam">head_index</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">[</span><span class="nam">head_index</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_w_in_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">        <span class="str">"""Gets the w_in matrix for a particular layer."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t">        <span class="nam">w_in</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.mlp.W_in"</span><span class="op">]</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t">        <span class="key">if</span> <span class="str">f"blocks.{layer_index}.ln2.w"</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">:</span>  <span class="com"># If fold_ln == False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">            <span class="nam">ln_2</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t">            <span class="key">return</span> <span class="nam">w_in</span> <span class="op">*</span> <span class="nam">ln_2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">        <span class="key">return</span> <span class="nam">w_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">    <span class="key">def</span> <span class="nam">_get_w_out_matrix</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">layer_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">        <span class="str">"""Gets the w_out matrix for a particular layer."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t">            <span class="num">0</span> <span class="op">&lt;=</span> <span class="nam">layer_index</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">f"Layer index must be between 0 and {self.cfg.n_layers-1} but got {layer_index}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">params</span><span class="op">[</span><span class="str">f"blocks.{layer_index}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -253,7 +252,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14___init___py.html b/_static/coverage/d_af97b5493da09a14___init___py.html
index 116f4a632..fcd15ded1 100644
--- a/_static/coverage/d_af97b5493da09a14___init___py.html
+++ b/_static/coverage/d_af97b5493da09a14___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_evals_py.html b/_static/coverage/d_af97b5493da09a14_evals_py.html
index de8ae4faa..65d0dfaac 100644
--- a/_static/coverage/d_af97b5493da09a14_evals_py.html
+++ b/_static/coverage/d_af97b5493da09a14_evals_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -427,7 +427,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_head_detector_py.html b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
index 1210eaa96..395e1148a 100644
--- a/_static/coverage/d_af97b5493da09a14_head_detector_py.html
+++ b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -361,7 +361,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_hook_points_py.html b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
index 4567796f8..bed02cd06 100644
--- a/_static/coverage/d_af97b5493da09a14_hook_points_py.html
+++ b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -523,261 +523,263 @@ <h2>
     <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t"><span class="str">            clear_contexts (bool): If True, clears hook contexts whenever hooks are reset. Default is</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t"><span class="str">                False.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t"><span class="str">            *model_args: Positional arguments for the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t"><span class="str">            **model_kwargs: Keyword arguments for the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t"><span class="str">        Note:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t"><span class="str">            If you want to use backward hooks, set `reset_hooks_end` to False, so the backward hooks</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t"><span class="str">            remain active. This function only runs a forward pass.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">bwd_hooks</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span> <span class="key">and</span> <span class="nam">reset_hooks_end</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">448&#x202F;&#x219B;&#x202F;449</span><span class="annotate long">line 448 didn't jump to line 449, because the condition on line 448 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">                <span class="str">"WARNING: Hooks will be reset at the end of run_with_hooks. This removes the backward hooks before a backward pass can occur."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="key">with</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hooks</span><span class="op">(</span><span class="nam">fwd_hooks</span><span class="op">,</span> <span class="nam">bwd_hooks</span><span class="op">,</span> <span class="nam">reset_hooks_end</span><span class="op">,</span> <span class="nam">clear_contexts</span><span class="op">)</span> <span class="key">as</span> <span class="nam">hooked_model</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">            <span class="key">return</span> <span class="nam">hooked_model</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span><span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">    <span class="key">def</span> <span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># TODO: unsure about whether or not this device typing is correct or not?</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">        <span class="str">"""Adds hooks to the model to cache activations. Note: It does NOT actually run the model to get activations, that must be done separately.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): Which activations to cache. Can be a list of strings (hook names) or a filter function mapping hook names to booleans. Defaults to lambda name: True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): Whether to also do backwards hooks. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t"><span class="str">            device (_type_, optional): The device to store on. Defaults to same device as model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): Whether to remove the batch dimension (only works for batch_size==1). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t"><span class="str">            cache (Optional[dict], optional): The cache to store activations in, a new dict is created by default. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t"><span class="str">            cache (dict): The cache where activations will be stored.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="key">if</span> <span class="nam">cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">        <span class="key">if</span> <span class="nam">names_filter</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">            <span class="nam">filter_str</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="op">==</span> <span class="nam">filter_str</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">            <span class="nam">filter_list</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">filter_list</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">        <span class="key">assert</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span><span class="op">,</span> <span class="str">"names_filter must be a callable"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">is_caching</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">        <span class="key">def</span> <span class="nam">save_hook</span><span class="op">(</span><span class="nam">tensor</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">hook</span><span class="op">:</span> <span class="nam">HookPoint</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">            <span class="key">assert</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">            <span class="key">if</span> <span class="nam">is_backward</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">                <span class="nam">hook_name</span> <span class="op">+=</span> <span class="str">"_grad"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">            <span class="key">if</span> <span class="nam">remove_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">                <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">                <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">hp</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">            <span class="key">if</span> <span class="nam">names_filter</span><span class="op">(</span><span class="nam">name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">                <span class="nam">hp</span><span class="op">.</span><span class="nam">add_hook</span><span class="op">(</span><span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">,</span> <span class="str">"fwd"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">                <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">                    <span class="nam">hp</span><span class="op">.</span><span class="nam">add_hook</span><span class="op">(</span><span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">,</span> <span class="str">"bwd"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">        <span class="key">return</span> <span class="nam">cache</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">        <span class="op">*</span><span class="nam">model_args</span><span class="op">:</span> <span class="nam">Any</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="nam">reset_hooks_end</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="nam">clear_contexts</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">:</span> <span class="nam">Any</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t"><span class="str">        Runs the model and returns the model output and a Cache object.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t"><span class="str">            *model_args: Positional arguments for the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): A filter for which activations to cache. Accepts None, str,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t"><span class="str">                list of str, or a function that takes a string and returns a bool. Defaults to None, which</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t"><span class="str">                means cache everything.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t"><span class="str">            device (str or torch.Device, optional): The device to cache activations on. Defaults to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t"><span class="str">                model device. WARNING: Setting a different device than the one used by the model leads to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t"><span class="str">                significant performance degradation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): If True, removes the batch dimension when caching. Only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t"><span class="str">                makes sense with batch_size=1 inputs. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): If True, calls backward on the model output and caches gradients</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t"><span class="str">                as well. Assumes that the model outputs a scalar (e.g., return_type="loss"). Custom loss</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t"><span class="str">                functions are not supported. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t"><span class="str">            reset_hooks_end (bool, optional): If True, removes all hooks added by this function at the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t"><span class="str">                end of the run. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t"><span class="str">            clear_contexts (bool, optional): If True, clears hook contexts whenever hooks are reset.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t"><span class="str">                Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t"><span class="str">                The slice to apply to the cache output. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t"><span class="str">            **model_kwargs: Keyword arguments for the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t"><span class="str">            tuple: A tuple containing the model output and a Cache object.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t"><span class="str">            **model_kwargs: Keyword arguments for the model's forward function. See your related</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t"><span class="str">                models forward pass for details as to what sort of arguments you can pass through.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t"><span class="str">        Note:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t"><span class="str">            If you want to use backward hooks, set `reset_hooks_end` to False, so the backward hooks</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t"><span class="str">            remain active. This function only runs a forward pass.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">        <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">bwd_hooks</span><span class="op">)</span> <span class="op">></span> <span class="num">0</span> <span class="key">and</span> <span class="nam">reset_hooks_end</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">449&#x202F;&#x219B;&#x202F;450</span><span class="annotate long">line 449 didn't jump to line 450, because the condition on line 449 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">                <span class="str">"WARNING: Hooks will be reset at the end of run_with_hooks. This removes the backward hooks before a backward pass can occur."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">        <span class="key">with</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hooks</span><span class="op">(</span><span class="nam">fwd_hooks</span><span class="op">,</span> <span class="nam">bwd_hooks</span><span class="op">,</span> <span class="nam">reset_hooks_end</span><span class="op">,</span> <span class="nam">clear_contexts</span><span class="op">)</span> <span class="key">as</span> <span class="nam">hooked_model</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">            <span class="key">return</span> <span class="nam">hooked_model</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span><span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">    <span class="key">def</span> <span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>  <span class="com"># TODO: unsure about whether or not this device typing is correct or not?</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">        <span class="str">"""Adds hooks to the model to cache activations. Note: It does NOT actually run the model to get activations, that must be done separately.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): Which activations to cache. Can be a list of strings (hook names) or a filter function mapping hook names to booleans. Defaults to lambda name: True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): Whether to also do backwards hooks. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t"><span class="str">            device (_type_, optional): The device to store on. Defaults to same device as model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): Whether to remove the batch dimension (only works for batch_size==1). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t"><span class="str">            cache (Optional[dict], optional): The cache to store activations in, a new dict is created by default. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t"><span class="str">            cache (dict): The cache where activations will be stored.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="key">if</span> <span class="nam">cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">        <span class="key">if</span> <span class="nam">names_filter</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">            <span class="nam">filter_str</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="op">==</span> <span class="nam">filter_str</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">            <span class="nam">filter_list</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">filter_list</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">        <span class="key">assert</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span><span class="op">,</span> <span class="str">"names_filter must be a callable"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">is_caching</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">        <span class="key">def</span> <span class="nam">save_hook</span><span class="op">(</span><span class="nam">tensor</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">hook</span><span class="op">:</span> <span class="nam">HookPoint</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">            <span class="key">assert</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">            <span class="key">if</span> <span class="nam">is_backward</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">                <span class="nam">hook_name</span> <span class="op">+=</span> <span class="str">"_grad"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">            <span class="key">if</span> <span class="nam">remove_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">                <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">                <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">hp</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">            <span class="key">if</span> <span class="nam">names_filter</span><span class="op">(</span><span class="nam">name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">                <span class="nam">hp</span><span class="op">.</span><span class="nam">add_hook</span><span class="op">(</span><span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">,</span> <span class="str">"fwd"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">                <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">                    <span class="nam">hp</span><span class="op">.</span><span class="nam">add_hook</span><span class="op">(</span><span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">,</span> <span class="str">"bwd"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">        <span class="key">return</span> <span class="nam">cache</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">    <span class="key">def</span> <span class="nam">run_with_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="op">*</span><span class="nam">model_args</span><span class="op">:</span> <span class="nam">Any</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="nam">reset_hooks_end</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="nam">clear_contexts</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">        <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">:</span> <span class="nam">Any</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t"><span class="str">        Runs the model and returns the model output and a Cache object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t"><span class="str">            *model_args: Positional arguments for the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): A filter for which activations to cache. Accepts None, str,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t"><span class="str">                list of str, or a function that takes a string and returns a bool. Defaults to None, which</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t"><span class="str">                means cache everything.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t"><span class="str">            device (str or torch.Device, optional): The device to cache activations on. Defaults to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t"><span class="str">                model device. WARNING: Setting a different device than the one used by the model leads to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t"><span class="str">                significant performance degradation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): If True, removes the batch dimension when caching. Only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t"><span class="str">                makes sense with batch_size=1 inputs. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): If True, calls backward on the model output and caches gradients</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t"><span class="str">                as well. Assumes that the model outputs a scalar (e.g., return_type="loss"). Custom loss</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t"><span class="str">                functions are not supported. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t"><span class="str">            reset_hooks_end (bool, optional): If True, removes all hooks added by this function at the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t"><span class="str">                end of the run. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t"><span class="str">            clear_contexts (bool, optional): If True, clears hook contexts whenever hooks are reset.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t"><span class="str">                Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t"><span class="str">            pos_slice:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t"><span class="str">                The slice to apply to the cache output. Defaults to None, do nothing.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t"><span class="str">            **model_kwargs: Keyword arguments for the model's forward function. See your related</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t"><span class="str">                models forward pass for details as to what sort of arguments you can pass through.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t"><span class="str">            tuple: A tuple containing the model output and a Cache object.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="nam">cache_dict</span><span class="op">,</span> <span class="nam">fwd</span><span class="op">,</span> <span class="nam">bwd</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">get_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">            <span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">            <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="key">with</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="nam">fwd_hooks</span><span class="op">=</span><span class="nam">fwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">            <span class="nam">bwd_hooks</span><span class="op">=</span><span class="nam">bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="nam">reset_hooks_end</span><span class="op">=</span><span class="nam">reset_hooks_end</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="nam">clear_contexts</span><span class="op">=</span><span class="nam">clear_contexts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">            <span class="nam">model_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">(</span><span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">567&#x202F;&#x219B;&#x202F;568</span><span class="annotate long">line 567 didn't jump to line 568, because the condition on line 567 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">                <span class="nam">model_out</span><span class="op">.</span><span class="nam">backward</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">        <span class="key">return</span> <span class="nam">model_out</span><span class="op">,</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">        <span class="nam">cache_dict</span><span class="op">,</span> <span class="nam">fwd</span><span class="op">,</span> <span class="nam">bwd</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">get_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">            <span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">            <span class="nam">pos_slice</span><span class="op">=</span><span class="nam">pos_slice</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">        <span class="key">with</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="nam">fwd_hooks</span><span class="op">=</span><span class="nam">fwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="nam">bwd_hooks</span><span class="op">=</span><span class="nam">bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">            <span class="nam">reset_hooks_end</span><span class="op">=</span><span class="nam">reset_hooks_end</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">            <span class="nam">clear_contexts</span><span class="op">=</span><span class="nam">clear_contexts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">            <span class="nam">model_out</span> <span class="op">=</span> <span class="nam">self</span><span class="op">(</span><span class="op">*</span><span class="nam">model_args</span><span class="op">,</span> <span class="op">**</span><span class="nam">model_kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">            <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">569&#x202F;&#x219B;&#x202F;570</span><span class="annotate long">line 569 didn't jump to line 570, because the condition on line 569 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">                <span class="nam">model_out</span><span class="op">.</span><span class="nam">backward</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">dict</span><span class="op">,</span> <span class="nam">list</span><span class="op">,</span> <span class="nam">list</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">        <span class="str">"""Creates hooks to cache activations. Note: It does not add the hooks to the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): Which activations to cache. Can be a list of strings (hook names) or a filter function mapping hook names to booleans. Defaults to lambda name: True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): Whether to also do backwards hooks. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t"><span class="str">            device (_type_, optional): The device to store on. Keeps on the same device as the layer if None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): Whether to remove the batch dimension (only works for batch_size==1). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t"><span class="str">            cache (Optional[dict], optional): The cache to store activations in, a new dict is created by default. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t"><span class="str">            cache (dict): The cache where activations will be stored.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">            fwd_hooks (list): The forward hooks.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t"><span class="str">            bwd_hooks (list): The backward hooks. Empty if incl_bwd is False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">        <span class="key">if</span> <span class="nam">cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">595&#x202F;&#x219B;&#x202F;598</span><span class="annotate long">line 595 didn't jump to line 598, because the condition on line 595 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">        <span class="key">return</span> <span class="nam">model_out</span><span class="op">,</span> <span class="nam">cache_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">    <span class="key">def</span> <span class="nam">get_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="nam">names_filter</span><span class="op">:</span> <span class="nam">NamesFilter</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">        <span class="nam">pos_slice</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Slice</span><span class="op">,</span> <span class="nam">SliceInput</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">dict</span><span class="op">,</span> <span class="nam">list</span><span class="op">,</span> <span class="nam">list</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">        <span class="str">"""Creates hooks to cache activations. Note: It does not add the hooks to the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t"><span class="str">            names_filter (NamesFilter, optional): Which activations to cache. Can be a list of strings (hook names) or a filter function mapping hook names to booleans. Defaults to lambda name: True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">            incl_bwd (bool, optional): Whether to also do backwards hooks. Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t"><span class="str">            device (_type_, optional): The device to store on. Keeps on the same device as the layer if None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t"><span class="str">            remove_batch_dim (bool, optional): Whether to remove the batch dimension (only works for batch_size==1). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t"><span class="str">            cache (Optional[dict], optional): The cache to store activations in, a new dict is created by default. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t"><span class="str">            cache (dict): The cache where activations will be stored.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">            fwd_hooks (list): The forward hooks.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t"><span class="str">            bwd_hooks (list): The backward hooks. Empty if incl_bwd is False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">        <span class="key">if</span> <span class="nam">cache</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">597&#x202F;&#x219B;&#x202F;600</span><span class="annotate long">line 597 didn't jump to line 600, because the condition on line 597 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">            <span class="nam">cache</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">        <span class="key">if</span> <span class="nam">names_filter</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">602&#x202F;&#x219B;&#x202F;603</span><span class="annotate long">line 602 didn't jump to line 603, because the condition on line 602 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">            <span class="nam">filter_str</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="op">==</span> <span class="nam">filter_str</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">            <span class="nam">filter_list</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">filter_list</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">        <span class="key">elif</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">608&#x202F;&#x219B;&#x202F;611</span><span class="annotate long">line 608 didn't jump to line 611, because the condition on line 608 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"names_filter must be a string, list of strings, or function"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">        <span class="key">assert</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span>  <span class="com"># Callable[[str], bool]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">is_caching</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">        <span class="nam">pos_slice</span> <span class="op">=</span> <span class="nam">Slice</span><span class="op">.</span><span class="nam">unwrap</span><span class="op">(</span><span class="nam">pos_slice</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">        <span class="key">if</span> <span class="nam">names_filter</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">604&#x202F;&#x219B;&#x202F;605</span><span class="annotate long">line 604 didn't jump to line 605, because the condition on line 604 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">            <span class="nam">filter_str</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="op">==</span> <span class="nam">filter_str</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">        <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">,</span> <span class="nam">list</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">            <span class="nam">filter_list</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">filter_list</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">        <span class="key">elif</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">610&#x202F;&#x219B;&#x202F;613</span><span class="annotate long">line 610 didn't jump to line 613, because the condition on line 610 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">            <span class="nam">names_filter</span> <span class="op">=</span> <span class="nam">names_filter</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"names_filter must be a string, list of strings, or function"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="key">assert</span> <span class="nam">callable</span><span class="op">(</span><span class="nam">names_filter</span><span class="op">)</span>  <span class="com"># Callable[[str], bool]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="key">def</span> <span class="nam">save_hook</span><span class="op">(</span><span class="nam">tensor</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">hook</span><span class="op">:</span> <span class="nam">HookPoint</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">            <span class="com"># for attention heads the pos dimension is the third from last</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">            <span class="key">if</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">618&#x202F;&#x219B;&#x202F;619</span><span class="annotate long">line 618 didn't jump to line 619, because the condition on line 618 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">                <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span><span class="str">"Hook should have been provided a name"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">            <span class="key">if</span> <span class="nam">is_backward</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">622&#x202F;&#x219B;&#x202F;623</span><span class="annotate long">line 622 didn't jump to line 623, because the condition on line 622 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">                <span class="nam">hook_name</span> <span class="op">+=</span> <span class="str">"_grad"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">            <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">            <span class="key">if</span> <span class="nam">remove_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">                <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">resid_stream</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">                <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_q"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_k"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_v"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_z"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_result"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">            <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">                <span class="nam">pos_dim</span> <span class="op">=</span> <span class="op">-</span><span class="num">3</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">                <span class="com"># for all other components the pos dimension is the second from last</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">                <span class="com"># including the attn scores where the dest token is the second from last</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">                <span class="nam">pos_dim</span> <span class="op">=</span> <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">641&#x202F;&#x219B;&#x202F;645</span><span class="annotate long">line 641 didn't jump to line 645</span></span></p>
-    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">                <span class="nam">tensor</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">>=</span> <span class="op">-</span><span class="nam">pos_dim</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">            <span class="op">)</span><span class="op">:</span>  <span class="com"># check if the residual stream has a pos dimension before trying to slice</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">                <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">resid_stream</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="nam">pos_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">            <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">resid_stream</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">        <span class="nam">fwd_hooks</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">        <span class="nam">bwd_hooks</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">_</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">            <span class="key">if</span> <span class="nam">names_filter</span><span class="op">(</span><span class="nam">name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">                <span class="nam">fwd_hooks</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="op">(</span><span class="nam">name</span><span class="op">,</span> <span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">                <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">652&#x202F;&#x219B;&#x202F;653</span><span class="annotate long">line 652 didn't jump to line 653, because the condition on line 652 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">                    <span class="nam">bwd_hooks</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="op">(</span><span class="nam">name</span><span class="op">,</span> <span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">        <span class="key">return</span> <span class="nam">cache</span><span class="op">,</span> <span class="nam">fwd_hooks</span><span class="op">,</span> <span class="nam">bwd_hooks</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">is_caching</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="key">def</span> <span class="nam">save_hook</span><span class="op">(</span><span class="nam">tensor</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="nam">hook</span><span class="op">:</span> <span class="nam">HookPoint</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">            <span class="com"># for attention heads the pos dimension is the third from last</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">            <span class="key">if</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">620&#x202F;&#x219B;&#x202F;621</span><span class="annotate long">line 620 didn't jump to line 621, because the condition on line 620 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">                <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span><span class="str">"Hook should have been provided a name"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">            <span class="nam">hook_name</span> <span class="op">=</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">            <span class="key">if</span> <span class="nam">is_backward</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">624&#x202F;&#x219B;&#x202F;625</span><span class="annotate long">line 624 didn't jump to line 625, because the condition on line 624 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">                <span class="nam">hook_name</span> <span class="op">+=</span> <span class="str">"_grad"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">            <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">tensor</span><span class="op">.</span><span class="nam">detach</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">            <span class="key">if</span> <span class="nam">remove_batch_dim</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">                <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">resid_stream</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">                <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_q"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_k"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_v"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_z"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">                <span class="key">or</span> <span class="nam">hook</span><span class="op">.</span><span class="nam">name</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"hook_result"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">            <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">                <span class="nam">pos_dim</span> <span class="op">=</span> <span class="op">-</span><span class="num">3</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">                <span class="com"># for all other components the pos dimension is the second from last</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">                <span class="com"># including the attn scores where the dest token is the second from last</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">                <span class="nam">pos_dim</span> <span class="op">=</span> <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">            <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">643&#x202F;&#x219B;&#x202F;647</span><span class="annotate long">line 643 didn't jump to line 647</span></span></p>
+    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">                <span class="nam">tensor</span><span class="op">.</span><span class="nam">dim</span><span class="op">(</span><span class="op">)</span> <span class="op">>=</span> <span class="op">-</span><span class="nam">pos_dim</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">            <span class="op">)</span><span class="op">:</span>  <span class="com"># check if the residual stream has a pos dimension before trying to slice</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">                <span class="nam">resid_stream</span> <span class="op">=</span> <span class="nam">pos_slice</span><span class="op">.</span><span class="nam">apply</span><span class="op">(</span><span class="nam">resid_stream</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="nam">pos_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">            <span class="nam">cache</span><span class="op">[</span><span class="nam">hook_name</span><span class="op">]</span> <span class="op">=</span> <span class="nam">resid_stream</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">        <span class="nam">fwd_hooks</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">        <span class="nam">bwd_hooks</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">_</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">            <span class="key">if</span> <span class="nam">names_filter</span><span class="op">(</span><span class="nam">name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">                <span class="nam">fwd_hooks</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="op">(</span><span class="nam">name</span><span class="op">,</span> <span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">False</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">                <span class="key">if</span> <span class="nam">incl_bwd</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">654&#x202F;&#x219B;&#x202F;655</span><span class="annotate long">line 654 didn't jump to line 655, because the condition on line 654 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">                    <span class="nam">bwd_hooks</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="op">(</span><span class="nam">name</span><span class="op">,</span> <span class="nam">partial</span><span class="op">(</span><span class="nam">save_hook</span><span class="op">,</span> <span class="nam">is_backward</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">    <span class="key">def</span> <span class="nam">cache_all</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">            <span class="str">"cache_all is deprecated and will eventually be removed, use add_caching_hooks or run_with_cache"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">667&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 667&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 667 didn't jump to the function exit, 2) line 667 didn't return from function 'cache_all', because </span></span></p>
-    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">=</span><span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">            <span class="nam">cache</span><span class="op">=</span><span class="nam">cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">=</span><span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">    <span class="key">def</span> <span class="nam">cache_some</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t">        <span class="nam">names</span><span class="op">:</span> <span class="nam">Callable</span><span class="op">[</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">bool</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="str">"""Cache a list of hook provided by names, Boolean function on names"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">            <span class="str">"cache_some is deprecated and will eventually be removed, use add_caching_hooks or run_with_cache"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">=</span><span class="nam">names</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">            <span class="nam">cache</span><span class="op">=</span><span class="nam">cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">=</span><span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t"><span class="com"># %%</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">        <span class="key">return</span> <span class="nam">cache</span><span class="op">,</span> <span class="nam">fwd_hooks</span><span class="op">,</span> <span class="nam">bwd_hooks</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="key">def</span> <span class="nam">cache_all</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">            <span class="str">"cache_all is deprecated and will eventually be removed, use add_caching_hooks or run_with_cache"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">669&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 669&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 669 didn't jump to the function exit, 2) line 669 didn't return from function 'cache_all', because </span></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">=</span><span class="key">lambda</span> <span class="nam">name</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">            <span class="nam">cache</span><span class="op">=</span><span class="nam">cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">=</span><span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">    <span class="key">def</span> <span class="nam">cache_some</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">        <span class="nam">cache</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">        <span class="nam">names</span><span class="op">:</span> <span class="nam">Callable</span><span class="op">[</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span><span class="op">,</span> <span class="nam">bool</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">        <span class="nam">incl_bwd</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">        <span class="nam">device</span><span class="op">:</span> <span class="nam">DeviceType</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="nam">remove_batch_dim</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">        <span class="str">"""Cache a list of hook provided by names, Boolean function on names"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">            <span class="str">"cache_some is deprecated and will eventually be removed, use add_caching_hooks or run_with_cache"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">add_caching_hooks</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">            <span class="nam">names_filter</span><span class="op">=</span><span class="nam">names</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">            <span class="nam">cache</span><span class="op">=</span><span class="nam">cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">            <span class="nam">incl_bwd</span><span class="op">=</span><span class="nam">incl_bwd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">            <span class="nam">remove_batch_dim</span><span class="op">=</span><span class="nam">remove_batch_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t"><span class="com"># %%</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -787,7 +789,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
index 079fdd6ee..371ec0a48 100644
--- a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
+++ b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
@@ -59,7 +59,7 @@ <h2>
             <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">208<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">112<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">52<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">51<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -296,1706 +296,1722 @@ <h2>
     <p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">    <span class="str">"Qwen/Qwen2-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t">    <span class="str">"Qwen/Qwen2-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t"><span class="str">"""Official model names for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t"><span class="com"># Model Aliases:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t"><span class="nam">MODEL_ALIASES</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L_v9_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l-pile"</span><span class="op">,</span> <span class="str">"solu-1l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L_v10_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l-pile"</span><span class="op">,</span> <span class="str">"solu-2l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L_v11_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l-pile"</span><span class="op">,</span> <span class="str">"solu-4l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L_v13_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l-pile"</span><span class="op">,</span> <span class="str">"solu-6l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L_v21_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l-pile"</span><span class="op">,</span> <span class="str">"solu-8l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L_v22_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l-pile"</span><span class="op">,</span> <span class="str">"solu-10l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L_v23_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l-pile"</span><span class="op">,</span> <span class="str">"solu-12l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l"</span><span class="op">,</span> <span class="str">"solu-1l-new"</span><span class="op">,</span> <span class="str">"solu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l"</span><span class="op">,</span> <span class="str">"solu-2l-new"</span><span class="op">,</span> <span class="str">"solu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-3l"</span><span class="op">,</span> <span class="str">"solu-3l-new"</span><span class="op">,</span> <span class="str">"solu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l"</span><span class="op">,</span> <span class="str">"solu-4l-new"</span><span class="op">,</span> <span class="str">"solu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t">    <span class="str">"NeelNanda/GELU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-1l"</span><span class="op">,</span> <span class="str">"gelu-1l-new"</span><span class="op">,</span> <span class="str">"gelu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t">    <span class="str">"NeelNanda/GELU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-2l"</span><span class="op">,</span> <span class="str">"gelu-2l-new"</span><span class="op">,</span> <span class="str">"gelu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t">    <span class="str">"NeelNanda/GELU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-3l"</span><span class="op">,</span> <span class="str">"gelu-3l-new"</span><span class="op">,</span> <span class="str">"gelu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t">    <span class="str">"NeelNanda/GELU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-4l"</span><span class="op">,</span> <span class="str">"gelu-4l-new"</span><span class="op">,</span> <span class="str">"gelu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">        <span class="str">"attn-only-1l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">        <span class="str">"attn-only-1l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">        <span class="str">"attn-only-1l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">        <span class="str">"attn-only-2l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">        <span class="str">"attn-only-2l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">        <span class="str">"attn-only-2l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">        <span class="str">"attn-only-3l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">        <span class="str">"attn-only-3l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">        <span class="str">"attn-only-3l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">        <span class="str">"attn-only-4l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">        <span class="str">"attn-only-4l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">        <span class="str">"attn-only-4l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-0.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-1.5B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-3B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-7B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-14B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-32B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t">    <span class="str">"Qwen/Qwen2.5-72B-Instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t">    <span class="str">"Qwen/QwQ-32B-Preview"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t"><span class="str">"""Official model names for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t"><span class="com"># Model Aliases:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t"><span class="nam">MODEL_ALIASES</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L_v9_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l-pile"</span><span class="op">,</span> <span class="str">"solu-1l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L_v10_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l-pile"</span><span class="op">,</span> <span class="str">"solu-2l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t259" href="#t259">259</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L_v11_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l-pile"</span><span class="op">,</span> <span class="str">"solu-4l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t260" href="#t260">260</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L_v13_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l-pile"</span><span class="op">,</span> <span class="str">"solu-6l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t261" href="#t261">261</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L_v21_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l-pile"</span><span class="op">,</span> <span class="str">"solu-8l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t262" href="#t262">262</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L_v22_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l-pile"</span><span class="op">,</span> <span class="str">"solu-10l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t263" href="#t263">263</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L_v23_old"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l-pile"</span><span class="op">,</span> <span class="str">"solu-12l-old"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t264" href="#t264">264</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-1l"</span><span class="op">,</span> <span class="str">"solu-1l-new"</span><span class="op">,</span> <span class="str">"solu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t265" href="#t265">265</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-2l"</span><span class="op">,</span> <span class="str">"solu-2l-new"</span><span class="op">,</span> <span class="str">"solu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t266" href="#t266">266</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-3l"</span><span class="op">,</span> <span class="str">"solu-3l-new"</span><span class="op">,</span> <span class="str">"solu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t267" href="#t267">267</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-4l"</span><span class="op">,</span> <span class="str">"solu-4l-new"</span><span class="op">,</span> <span class="str">"solu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t268" href="#t268">268</a></span><span class="t">    <span class="str">"NeelNanda/GELU_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-1l"</span><span class="op">,</span> <span class="str">"gelu-1l-new"</span><span class="op">,</span> <span class="str">"gelu-1l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t269" href="#t269">269</a></span><span class="t">    <span class="str">"NeelNanda/GELU_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-2l"</span><span class="op">,</span> <span class="str">"gelu-2l-new"</span><span class="op">,</span> <span class="str">"gelu-2l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t270" href="#t270">270</a></span><span class="t">    <span class="str">"NeelNanda/GELU_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-3l"</span><span class="op">,</span> <span class="str">"gelu-3l-new"</span><span class="op">,</span> <span class="str">"gelu-3l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t271" href="#t271">271</a></span><span class="t">    <span class="str">"NeelNanda/GELU_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gelu-4l"</span><span class="op">,</span> <span class="str">"gelu-4l-new"</span><span class="op">,</span> <span class="str">"gelu-4l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t272" href="#t272">272</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_1L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t273" href="#t273">273</a></span><span class="t">        <span class="str">"attn-only-1l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t274" href="#t274">274</a></span><span class="t">        <span class="str">"attn-only-1l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t275" href="#t275">275</a></span><span class="t">        <span class="str">"attn-only-1l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t276" href="#t276">276</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L768W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l"</span><span class="op">,</span> <span class="str">"solu-6l-new"</span><span class="op">,</span> <span class="str">"solu-6l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L1024W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l"</span><span class="op">,</span> <span class="str">"solu-8l-new"</span><span class="op">,</span> <span class="str">"solu-8l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L1280W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l"</span><span class="op">,</span> <span class="str">"solu-10l-new"</span><span class="op">,</span> <span class="str">"solu-10l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L1536W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l"</span><span class="op">,</span> <span class="str">"solu-12l-new"</span><span class="op">,</span> <span class="str">"solu-12l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t">    <span class="str">"NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t">        <span class="str">"attn-only-2l-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t">        <span class="str">"attn-only-2l-shortformer-6b-big-lr"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t">        <span class="str">"attn-only-2l-induction-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t">        <span class="str">"attn-only-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t277" href="#t277">277</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_2L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t278" href="#t278">278</a></span><span class="t">        <span class="str">"attn-only-2l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t279" href="#t279">279</a></span><span class="t">        <span class="str">"attn-only-2l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t280" href="#t280">280</a></span><span class="t">        <span class="str">"attn-only-2l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t281" href="#t281">281</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t282" href="#t282">282</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_3L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t283" href="#t283">283</a></span><span class="t">        <span class="str">"attn-only-3l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t284" href="#t284">284</a></span><span class="t">        <span class="str">"attn-only-3l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t285" href="#t285">285</a></span><span class="t">        <span class="str">"attn-only-3l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t286" href="#t286">286</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">        <span class="str">"solu-1l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">        <span class="str">"solu-1l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">        <span class="str">"solu-1l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t287" href="#t287">287</a></span><span class="t">    <span class="str">"NeelNanda/Attn_Only_4L512W_C4_Code"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t288" href="#t288">288</a></span><span class="t">        <span class="str">"attn-only-4l"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t289" href="#t289">289</a></span><span class="t">        <span class="str">"attn-only-4l-new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t290" href="#t290">290</a></span><span class="t">        <span class="str">"attn-only-4l-c4-code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t291" href="#t291">291</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">        <span class="str">"solu-4l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">        <span class="str">"solu-4l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">        <span class="str">"solu-4l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">    <span class="str">"EleutherAI/pythia-14m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="str">"pythia-14m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">    <span class="str">"EleutherAI/pythia-31m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">        <span class="str">"pythia-31m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">        <span class="str">"pythia-70m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">        <span class="str">"pythia"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">        <span class="str">"pythia-19m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="str">"pythia-160m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">        <span class="str">"pythia-125m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">        <span class="str">"pythia-410m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">        <span class="str">"pythia-350m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="str">"pythia-1b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="str">"pythia-800m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t292" href="#t292">292</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_6L768W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-6l"</span><span class="op">,</span> <span class="str">"solu-6l-new"</span><span class="op">,</span> <span class="str">"solu-6l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t293" href="#t293">293</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_8L1024W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-8l"</span><span class="op">,</span> <span class="str">"solu-8l-new"</span><span class="op">,</span> <span class="str">"solu-8l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t294" href="#t294">294</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_10L1280W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-10l"</span><span class="op">,</span> <span class="str">"solu-10l-new"</span><span class="op">,</span> <span class="str">"solu-10l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t295" href="#t295">295</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_12L1536W_C4_Code"</span><span class="op">:</span> <span class="op">[</span><span class="str">"solu-12l"</span><span class="op">,</span> <span class="str">"solu-12l-new"</span><span class="op">,</span> <span class="str">"solu-12l-c4-code"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t296" href="#t296">296</a></span><span class="t">    <span class="str">"NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">        <span class="str">"attn-only-2l-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">        <span class="str">"attn-only-2l-shortformer-6b-big-lr"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">        <span class="str">"attn-only-2l-induction-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">        <span class="str">"attn-only-demo"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_1L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">        <span class="str">"solu-1l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">        <span class="str">"solu-1l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">        <span class="str">"solu-1l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">    <span class="str">"NeelNanda/SoLU_4L512W_Wiki_Finetune"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">        <span class="str">"solu-4l-wiki"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">        <span class="str">"solu-4l-wiki-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">        <span class="str">"solu-4l-finetune"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">    <span class="str">"EleutherAI/pythia-14m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">        <span class="str">"pythia-14m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">    <span class="str">"EleutherAI/pythia-31m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">        <span class="str">"pythia-31m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">        <span class="str">"pythia-70m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="str">"pythia"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="str">"pythia-19m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="str">"pythia-1.4b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="str">"pythia-1.3b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="str">"pythia-160m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="str">"pythia-125m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="str">"pythia-2.8b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="str">"pythia-2.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="str">"pythia-410m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="str">"pythia-350m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">        <span class="str">"pythia-6.9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">        <span class="str">"pythia-6.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">        <span class="str">"pythia-1b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">        <span class="str">"pythia-800m"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">        <span class="str">"pythia-12b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">        <span class="str">"pythia-13b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">        <span class="str">"pythia-1.4b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">        <span class="str">"pythia-1.3b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">        <span class="str">"pythia-70m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">        <span class="str">"pythia-19m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">        <span class="str">"pythia-2.8b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">        <span class="str">"pythia-2.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">        <span class="str">"pythia-160m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">        <span class="str">"pythia-125m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">        <span class="str">"pythia-6.9b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">        <span class="str">"pythia-6.7b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">        <span class="str">"pythia-410m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">        <span class="str">"pythia-350m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">        <span class="str">"pythia-12b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">        <span class="str">"pythia-13b"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">        <span class="str">"pythia-1b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">        <span class="str">"pythia-800m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">        <span class="str">"pythia-70m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">        <span class="str">"pythia-19m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">        <span class="str">"pythia-160m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">        <span class="str">"pythia-125m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">        <span class="str">"pythia-410m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">        <span class="str">"pythia-350m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">        <span class="str">"pythia-1b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="str">"pythia-800m-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">        <span class="str">"pythia-12b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">        <span class="str">"pythia-13b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">        <span class="str">"pythia-70m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="str">"pythia-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">        <span class="str">"pythia-19m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="str">"pythia-160m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">        <span class="str">"pythia-125m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="str">"pythia-410m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">        <span class="str">"pythia-350m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">        <span class="str">"pythia-1b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="str">"pythia-800m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">        <span class="str">"pythia-12b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="str">"pythia-13b-deduped"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">        <span class="str">"pythia-70m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">        <span class="str">"pythia-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="str">"pythia-19m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">        <span class="str">"pythia-1.4b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="str">"pythia-1.3b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">        <span class="str">"pythia-160m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="str">"pythia-125m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="str">"pythia-2.8b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="str">"pythia-2.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="str">"pythia-410m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="str">"pythia-350m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">        <span class="str">"pythia-6.9b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">        <span class="str">"pythia-6.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">        <span class="str">"pythia-1b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">        <span class="str">"pythia-800m-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="str">"pythia-12b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="str">"pythia-13b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="str">"pythia-1.4b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="str">"pythia-1.3b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="str">"pythia-70m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="str">"pythia-19m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="str">"pythia-2.8b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="str">"pythia-2.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="str">"pythia-160m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">        <span class="str">"pythia-125m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="str">"pythia-6.9b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">        <span class="str">"pythia-6.7b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">        <span class="str">"pythia-410m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="str">"pythia-350m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">        <span class="str">"pythia-12b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="str">"pythia-13b-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="str">"pythia-1b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="str">"pythia-800m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">    <span class="str">"EleutherAI/pythia-70m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="str">"pythia-70m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="str">"EleutherAI/pythia-19m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="str">"pythia-19m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="str">"pythia-160m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="str">"pythia-125m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">    <span class="str">"EleutherAI/pythia-410m-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">        <span class="str">"pythia-410m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="str">"EleutherAI/pythia-350m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">        <span class="str">"pythia-350m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">        <span class="str">"pythia-1b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="str">"EleutherAI/pythia-800m-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="str">"pythia-800m-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="str">"pythia-12b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="str">"pythia-13b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">    <span class="str">"EleutherAI/pythia-1.4b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">        <span class="str">"pythia-1.4b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">        <span class="str">"EleutherAI/pythia-1.3b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="str">"pythia-1.3b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">        <span class="str">"pythia-160m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="str">"pythia-125m-seed1"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">    <span class="str">"EleutherAI/pythia-2.8b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">        <span class="str">"pythia-2.8b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="str">"EleutherAI/pythia-2.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="str">"pythia-2.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed2"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="str">"pythia-160m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="str">"pythia-125m-seed2"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">    <span class="str">"EleutherAI/pythia-6.9b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="str">"pythia-6.9b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="str">"EleutherAI/pythia-6.7b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="str">"pythia-6.7b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed3"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="str">"pythia-160m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">        <span class="str">"pythia-125m-seed3"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="str">"EleutherAI/pythia-12b-deduped-v0"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="str">"pythia-12b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="str">"EleutherAI/pythia-13b-deduped-v0"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">        <span class="str">"pythia-13b-deduped-v0"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">    <span class="str">"gpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt2-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">    <span class="str">"distilgpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"distillgpt2"</span><span class="op">,</span> <span class="str">"distill-gpt2"</span><span class="op">,</span> <span class="str">"distil-gpt2"</span><span class="op">,</span> <span class="str">"gpt2-xs"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">    <span class="str">"facebook/opt-125m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-125m"</span><span class="op">,</span> <span class="str">"opt-small"</span><span class="op">,</span> <span class="str">"opt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">    <span class="str">"facebook/opt-1.3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-1.3b"</span><span class="op">,</span> <span class="str">"opt-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">    <span class="str">"facebook/opt-2.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-2.7b"</span><span class="op">,</span> <span class="str">"opt-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">    <span class="str">"facebook/opt-6.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-6.7b"</span><span class="op">,</span> <span class="str">"opt-xl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">    <span class="str">"facebook/opt-13b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-13b"</span><span class="op">,</span> <span class="str">"opt-xxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">    <span class="str">"facebook/opt-30b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-30b"</span><span class="op">,</span> <span class="str">"opt-xxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">    <span class="str">"facebook/opt-66b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-66b"</span><span class="op">,</span> <span class="str">"opt-xxxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-125M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-125M"</span><span class="op">,</span> <span class="str">"gpt-neo-small"</span><span class="op">,</span> <span class="str">"neo-small"</span><span class="op">,</span> <span class="str">"neo"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-1.3B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-1.3B"</span><span class="op">,</span> <span class="str">"gpt-neo-medium"</span><span class="op">,</span> <span class="str">"neo-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-2.7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-2.7B"</span><span class="op">,</span> <span class="str">"gpt-neo-large"</span><span class="op">,</span> <span class="str">"neo-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">    <span class="str">"EleutherAI/gpt-j-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-j-6B"</span><span class="op">,</span> <span class="str">"gpt-j"</span><span class="op">,</span> <span class="str">"gptj"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neox-20b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neox-20b"</span><span class="op">,</span> <span class="str">"gpt-neox"</span><span class="op">,</span> <span class="str">"neox"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">    <span class="str">"stanford-crfm/alias-gpt2-small-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">        <span class="str">"stanford-gpt2-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">        <span class="str">"alias-gpt2-small-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">        <span class="str">"gpt2-mistral-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">        <span class="str">"gpt2-stanford-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">    <span class="str">"stanford-crfm/battlestar-gpt2-small-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">        <span class="str">"stanford-gpt2-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">        <span class="str">"battlestar-gpt2-small-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">    <span class="str">"stanford-crfm/caprica-gpt2-small-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">        <span class="str">"stanford-gpt2-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">        <span class="str">"caprica-gpt2-small-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">        <span class="str">"gpt2-mistral-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">        <span class="str">"gpt2-stanford-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">    <span class="str">"stanford-crfm/darkmatter-gpt2-small-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="str">"stanford-gpt2-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">        <span class="str">"darkmatter-gpt2-small-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">    <span class="str">"stanford-crfm/expanse-gpt2-small-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="str">"stanford-gpt2-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">        <span class="str">"expanse-gpt2-small-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">    <span class="str">"stanford-crfm/arwen-gpt2-medium-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">        <span class="str">"arwen-gpt2-medium-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">        <span class="str">"gpt2-medium-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">    <span class="str">"stanford-crfm/beren-gpt2-medium-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">        <span class="str">"beren-gpt2-medium-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">        <span class="str">"gpt2-medium-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">    <span class="str">"stanford-crfm/celebrimbor-gpt2-medium-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">        <span class="str">"celebrimbor-gpt2-medium-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">    <span class="str">"stanford-crfm/durin-gpt2-medium-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">        <span class="str">"durin-gpt2-medium-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">        <span class="str">"gpt2-medium-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">    <span class="str">"stanford-crfm/eowyn-gpt2-medium-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">        <span class="str">"eowyn-gpt2-medium-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">        <span class="str">"gpt2-medium-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">    <span class="str">"ArthurConmy/redwood_attn_2l"</span><span class="op">:</span> <span class="op">[</span><span class="str">"redwood_attn_2l"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-13b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-30b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-65b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">        <span class="str">"Llama-2-7b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="str">"Llama-2-13b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">        <span class="str">"pythia-160m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed1"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">        <span class="str">"pythia-125m-seed1"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed2"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">        <span class="str">"pythia-160m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">        <span class="str">"pythia-125m-seed2"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">    <span class="str">"EleutherAI/pythia-160m-seed3"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">        <span class="str">"pythia-160m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">        <span class="str">"EleutherAI/pythia-125m-seed3"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">        <span class="str">"pythia-125m-seed3"</span><span class="op">,</span>  <span class="com"># EleutherAI renamed this model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">    <span class="str">"gpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt2-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">    <span class="str">"distilgpt2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"distillgpt2"</span><span class="op">,</span> <span class="str">"distill-gpt2"</span><span class="op">,</span> <span class="str">"distil-gpt2"</span><span class="op">,</span> <span class="str">"gpt2-xs"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">    <span class="str">"facebook/opt-125m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-125m"</span><span class="op">,</span> <span class="str">"opt-small"</span><span class="op">,</span> <span class="str">"opt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">    <span class="str">"facebook/opt-1.3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-1.3b"</span><span class="op">,</span> <span class="str">"opt-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">    <span class="str">"facebook/opt-2.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-2.7b"</span><span class="op">,</span> <span class="str">"opt-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">    <span class="str">"facebook/opt-6.7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-6.7b"</span><span class="op">,</span> <span class="str">"opt-xl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">    <span class="str">"facebook/opt-13b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-13b"</span><span class="op">,</span> <span class="str">"opt-xxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">    <span class="str">"facebook/opt-30b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-30b"</span><span class="op">,</span> <span class="str">"opt-xxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">    <span class="str">"facebook/opt-66b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"opt-66b"</span><span class="op">,</span> <span class="str">"opt-xxxxl"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-125M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-125M"</span><span class="op">,</span> <span class="str">"gpt-neo-small"</span><span class="op">,</span> <span class="str">"neo-small"</span><span class="op">,</span> <span class="str">"neo"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-1.3B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-1.3B"</span><span class="op">,</span> <span class="str">"gpt-neo-medium"</span><span class="op">,</span> <span class="str">"neo-medium"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neo-2.7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neo-2.7B"</span><span class="op">,</span> <span class="str">"gpt-neo-large"</span><span class="op">,</span> <span class="str">"neo-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">    <span class="str">"EleutherAI/gpt-j-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-j-6B"</span><span class="op">,</span> <span class="str">"gpt-j"</span><span class="op">,</span> <span class="str">"gptj"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">    <span class="str">"EleutherAI/gpt-neox-20b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gpt-neox-20b"</span><span class="op">,</span> <span class="str">"gpt-neox"</span><span class="op">,</span> <span class="str">"neox"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">    <span class="str">"stanford-crfm/alias-gpt2-small-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">        <span class="str">"stanford-gpt2-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">        <span class="str">"alias-gpt2-small-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="str">"gpt2-mistral-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">        <span class="str">"gpt2-stanford-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">    <span class="str">"stanford-crfm/battlestar-gpt2-small-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="str">"stanford-gpt2-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="str">"battlestar-gpt2-small-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="str">"gpt2-mistral-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">    <span class="str">"stanford-crfm/caprica-gpt2-small-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">        <span class="str">"stanford-gpt2-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">        <span class="str">"caprica-gpt2-small-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">        <span class="str">"gpt2-mistral-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="str">"gpt2-stanford-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">    <span class="str">"stanford-crfm/darkmatter-gpt2-small-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="str">"stanford-gpt2-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">        <span class="str">"darkmatter-gpt2-small-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">        <span class="str">"gpt2-mistral-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">    <span class="str">"stanford-crfm/expanse-gpt2-small-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">        <span class="str">"stanford-gpt2-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">        <span class="str">"expanse-gpt2-small-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="str">"gpt2-mistral-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">    <span class="str">"stanford-crfm/arwen-gpt2-medium-x21"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">        <span class="str">"arwen-gpt2-medium-x21"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">        <span class="str">"gpt2-medium-small-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-a"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">    <span class="str">"stanford-crfm/beren-gpt2-medium-x49"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">        <span class="str">"beren-gpt2-medium-x49"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">        <span class="str">"gpt2-medium-small-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">    <span class="str">"stanford-crfm/celebrimbor-gpt2-medium-x81"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">        <span class="str">"celebrimbor-gpt2-medium-x81"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">        <span class="str">"gpt2-medium-small-c"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">    <span class="str">"stanford-crfm/durin-gpt2-medium-x343"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">        <span class="str">"durin-gpt2-medium-x343"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="str">"gpt2-medium-small-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-d"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">    <span class="str">"stanford-crfm/eowyn-gpt2-medium-x777"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">        <span class="str">"stanford-gpt2-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">        <span class="str">"eowyn-gpt2-medium-x777"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="str">"gpt2-medium-small-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">        <span class="str">"gpt2-stanford-medium-e"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-70b-chat-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-70b-chat"</span><span class="op">,</span> <span class="str">"meta-llama-2-70b-chat-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"CodeLlamallama-2-7b"</span><span class="op">,</span> <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">        <span class="str">"CodeLlama-7b-python"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="str">"CodeLlama-7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">    <span class="str">"ArthurConmy/redwood_attn_2l"</span><span class="op">:</span> <span class="op">[</span><span class="str">"redwood_attn_2l"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-13b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-30b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"llama-65b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">        <span class="str">"Llama-2-7b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-7b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">:</span> <span class="op">[</span><span class="str">"othello-gpt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">        <span class="str">"stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">        <span class="str">"stablelm-base-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">        <span class="str">"stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">        <span class="str">"stablelm-base-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">        <span class="str">"stablelm-tuned-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">        <span class="str">"stablelm-tuned-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b-instruct"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-nemo-base-2407"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mixtral"</span><span class="op">,</span> <span class="str">"mixtral-8x7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="str">"mixtral-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="str">"mixtral-8x7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-560m"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b7"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-3b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-7b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">:</span> <span class="op">[</span><span class="str">"santacoder"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1_5"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-2"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-3"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b"</span><span class="op">,</span> <span class="str">"Yi-6B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b"</span><span class="op">,</span> <span class="str">"Yi-34B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b-chat"</span><span class="op">,</span> <span class="str">"Yi-6B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b-chat"</span><span class="op">,</span> <span class="str">"Yi-34B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-base"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mGPT"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t"><span class="str">"""Model aliases for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t"><span class="nam">NON_HF_HOSTED_MODEL_NAMES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t"><span class="str">"""Official model names for models not hosted on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t"><span class="com"># Sets a default model alias, by convention the first one in the model alias table, else the official name if it has no aliases</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t"><span class="nam">DEFAULT_MODEL_ALIASES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">    <span class="nam">MODEL_ALIASES</span><span class="op">[</span><span class="nam">name</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="key">if</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">MODEL_ALIASES</span> <span class="key">else</span> <span class="nam">name</span> <span class="key">for</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="nam">NEED_REMOTE_CODE_MODELS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t">    <span class="str">"Qwen/Qwen-"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t"><span class="key">def</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t"><span class="str">    Converts OFFICIAL_MODEL_NAMES (the list of actual model names on</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t"><span class="str">    HuggingFace) and MODEL_ALIASES (a dictionary mapping official model names to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t"><span class="str">    aliases) into a dictionary mapping all aliases to the official model name.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">    <span class="key">for</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">        <span class="nam">aliases</span> <span class="op">=</span> <span class="nam">MODEL_ALIASES</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">[</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">        <span class="key">for</span> <span class="nam">alias</span> <span class="key">in</span> <span class="nam">aliases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">            <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">alias</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">        <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t">    <span class="key">return</span> <span class="nam">model_alias_map</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">        <span class="str">"Llama-2-13b-chat"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">        <span class="str">"meta-llama/Llama-2-13b-chat-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">    <span class="str">"meta-llama/Llama-2-70b-chat-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"Llama-2-70b-chat"</span><span class="op">,</span> <span class="str">"meta-llama-2-70b-chat-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">:</span> <span class="op">[</span><span class="str">"CodeLlamallama-2-7b"</span><span class="op">,</span> <span class="str">"codellama/CodeLlama-7b-hf"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t">        <span class="str">"CodeLlama-7b-python"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Python-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">    <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t">        <span class="str">"CodeLlama-7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t">        <span class="str">"codellama/CodeLlama-7b-Instruct-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t">    <span class="str">"Baidicoot/Othello-GPT-Transformer-Lens"</span><span class="op">:</span> <span class="op">[</span><span class="str">"othello-gpt"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-1M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-3M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-3M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-8M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-8M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-28M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-28M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instuct-1Layer-21M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-1L-21M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">    <span class="str">"roneneldan/TinyStories-Instruct-2Layers-33M"</span><span class="op">:</span> <span class="op">[</span><span class="str">"tiny-stories-instruct-2L-33M"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">        <span class="str">"stablelm-base-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">        <span class="str">"stablelm-base-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="str">"stabilityai/stablelm-base-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="str">"stablelm-base-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">        <span class="str">"stablelm-base-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-3b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">        <span class="str">"stablelm-tuned-3b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">    <span class="str">"stabilityai/stablelm-tuned-alpha-7b"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">        <span class="str">"stablelm-tuned-alpha-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="str">"stablelm-tuned-7b"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">    <span class="str">"mistralai/Mistral-7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-7b-instruct"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t">    <span class="str">"mistralai/Mistral-Nemo-Base-2407"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mistral-nemo-base-2407"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-v0.1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mixtral"</span><span class="op">,</span> <span class="str">"mixtral-8x7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t">    <span class="str">"mistralai/Mixtral-8x7B-Instruct-v0.1"</span><span class="op">:</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">        <span class="str">"mixtral-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t">        <span class="str">"mixtral-8x7b-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">    <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t">    <span class="str">"bigscience/bloom-560m"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-560m"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">    <span class="str">"bigscience/bloom-1b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">    <span class="str">"bigscience/bloom-1b7"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-1b7"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t">    <span class="str">"bigscience/bloom-3b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-3b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t">    <span class="str">"bigscience/bloom-7b1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"bloom-7b1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">:</span> <span class="op">[</span><span class="str">"santacoder"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">    <span class="str">"Qwen/Qwen-1_8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">    <span class="str">"Qwen/Qwen-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">    <span class="str">"Qwen/Qwen-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-0.5B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-0.5b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-1.8B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-1.8b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-4B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-4b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-7B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-7b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">    <span class="str">"Qwen/Qwen1.5-14B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"qwen1.5-14b-chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">    <span class="str">"microsoft/phi-1"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">    <span class="str">"microsoft/phi-1_5"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-1_5"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-2"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">:</span> <span class="op">[</span><span class="str">"phi-3"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="str">"google/gemma-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">    <span class="str">"google/gemma-7b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">    <span class="str">"google/gemma-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">    <span class="str">"google/gemma-7b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-7b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">    <span class="str">"google/gemma-2-2b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">    <span class="str">"google/gemma-2-9b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">    <span class="str">"google/gemma-2-27b"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t">    <span class="str">"google/gemma-2-2b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-2b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t">    <span class="str">"google/gemma-2-9b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-9b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t">    <span class="str">"google/gemma-2-27b-it"</span><span class="op">:</span> <span class="op">[</span><span class="str">"gemma-2-27b-it"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t">    <span class="str">"01-ai/Yi-6B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b"</span><span class="op">,</span> <span class="str">"Yi-6B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t">    <span class="str">"01-ai/Yi-34B"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b"</span><span class="op">,</span> <span class="str">"Yi-34B"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">    <span class="str">"01-ai/Yi-6B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-6b-chat"</span><span class="op">,</span> <span class="str">"Yi-6B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t">    <span class="str">"01-ai/Yi-34B-Chat"</span><span class="op">:</span> <span class="op">[</span><span class="str">"yi-34b-chat"</span><span class="op">,</span> <span class="str">"Yi-34B-Chat"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">    <span class="str">"google-t5/t5-small"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-small"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">    <span class="str">"google-t5/t5-base"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-base"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t">    <span class="str">"google-t5/t5-large"</span><span class="op">:</span> <span class="op">[</span><span class="str">"t5-large"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t">    <span class="str">"ai-forever/mGPT"</span><span class="op">:</span> <span class="op">[</span><span class="str">"mGPT"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="str">"""Model aliases for models on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t"><span class="nam">NON_HF_HOSTED_MODEL_NAMES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t">    <span class="str">"llama-7b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t">    <span class="str">"llama-13b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">    <span class="str">"llama-30b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">    <span class="str">"llama-65b-hf"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t"><span class="str">"""Official model names for models not hosted on HuggingFace."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t"><span class="com"># Sets a default model alias, by convention the first one in the model alias table, else the official name if it has no aliases</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t"><span class="nam">DEFAULT_MODEL_ALIASES</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">    <span class="nam">MODEL_ALIASES</span><span class="op">[</span><span class="nam">name</span><span class="op">]</span><span class="op">[</span><span class="num">0</span><span class="op">]</span> <span class="key">if</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">MODEL_ALIASES</span> <span class="key">else</span> <span class="nam">name</span> <span class="key">for</span> <span class="nam">name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t"><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t"><span class="nam">NEED_REMOTE_CODE_MODELS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">    <span class="str">"bigcode/santacoder"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">    <span class="str">"Qwen/Qwen-"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">    <span class="str">"microsoft/phi-2"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">    <span class="str">"microsoft/Phi-3-mini-4k-instruct"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t"><span class="key">def</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t"><span class="key">def</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t"><span class="str">    Returns the official model name for a given model name (or alias).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_alias_map</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">707&#x202F;&#x219B;&#x202F;708</span><span class="annotate long">line 707 didn't jump to line 708, because the condition on line 707 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">            <span class="str">f"{model_name} not found. Valid official model names (excl aliases): {OFFICIAL_MODEL_NAMES}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">    <span class="key">return</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t"><span class="str">    Returns the model config for a HuggingFace model, converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t"><span class="str">    Takes the official_model_name as an input.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">    <span class="com"># In case the user passed in an alias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span> <span class="op">/</span> <span class="str">"config.json"</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">722&#x202F;&#x219B;&#x202F;723</span><span class="annotate long">line 722 didn't jump to line 723, because the condition on line 722 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">"Loading model config from local directory"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t"><span class="str">    Converts OFFICIAL_MODEL_NAMES (the list of actual model names on</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t"><span class="str">    HuggingFace) and MODEL_ALIASES (a dictionary mapping official model names to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t"><span class="str">    aliases) into a dictionary mapping all aliases to the official model name.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">    <span class="key">for</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">OFFICIAL_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="nam">aliases</span> <span class="op">=</span> <span class="nam">MODEL_ALIASES</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">[</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t710" href="#t710">710</a></span><span class="t">        <span class="key">for</span> <span class="nam">alias</span> <span class="key">in</span> <span class="nam">aliases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t711" href="#t711">711</a></span><span class="t">            <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">alias</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t712" href="#t712">712</a></span><span class="t">        <span class="nam">model_alias_map</span><span class="op">[</span><span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t713" href="#t713">713</a></span><span class="t">    <span class="key">return</span> <span class="nam">model_alias_map</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t714" href="#t714">714</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t715" href="#t715">715</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t716" href="#t716">716</a></span><span class="t"><span class="key">def</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t717" href="#t717">717</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t718" href="#t718">718</a></span><span class="t"><span class="str">    Returns the official model name for a given model name (or alias).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t719" href="#t719">719</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t720" href="#t720">720</a></span><span class="t">    <span class="nam">model_alias_map</span> <span class="op">=</span> <span class="nam">make_model_alias_map</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t721" href="#t721">721</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_alias_map</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t722" href="#t722">722</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">722&#x202F;&#x219B;&#x202F;723</span><span class="annotate long">line 722 didn't jump to line 723, because the condition on line 722 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t723" href="#t723">723</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t724" href="#t724">724</a></span><span class="t">            <span class="str">f"{model_name} not found. Valid official model names (excl aliases): {OFFICIAL_MODEL_NAMES}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t725" href="#t725">725</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t726" href="#t726">726</a></span><span class="t">    <span class="key">return</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t727" href="#t727">727</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t">    <span class="com"># Load HuggingFace model config</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t">    <span class="key">if</span> <span class="str">"llama"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">729&#x202F;&#x219B;&#x202F;730</span><span class="annotate long">line 729 didn't jump to line 730, because the condition on line 729 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"LlamaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma-2"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">731&#x202F;&#x219B;&#x202F;732</span><span class="annotate long">line 731 didn't jump to line 732, because the condition on line 731 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"Gemma2ForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">733&#x202F;&#x219B;&#x202F;734</span><span class="annotate long">line 733 didn't jump to line 734, because the condition on line 733 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"GemmaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">        <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">        <span class="nam">hf_config</span> <span class="op">=</span> <span class="nam">AutoConfig</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">            <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">architectures</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">744&#x202F;&#x219B;&#x202F;747</span><span class="annotate long">line 744 didn't jump to line 747</span></span></p>
-    <p class="pln"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"codellama"</span><span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture CodeLlama and Llama-2</span>&nbsp;</span><span class="r"><span class="annotate short">764&#x202F;&#x219B;&#x202F;765</span><span class="annotate long">line 764 didn't jump to line 765</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">1000000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">        <span class="key">if</span> <span class="str">"python"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="com"># The vocab size of python version of CodeLlama-7b is 32000</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span> <span class="op">=</span> <span class="num">32000</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">785&#x202F;&#x219B;&#x202F;788</span><span class="annotate long">line 785 didn't jump to line 788</span></span></p>
-    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">5120</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">13824</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-30b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">805&#x202F;&#x219B;&#x202F;806</span><span class="annotate long">line 805 didn't jump to line 806</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">6656</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">17920</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">60</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-65b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">823&#x202F;&#x219B;&#x202F;824</span><span class="annotate long">line 823 didn't jump to line 824</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">22016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-2-70b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">841&#x202F;&#x219B;&#x202F;842</span><span class="annotate long">line 841 didn't jump to line 842</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">860&#x202F;&#x219B;&#x202F;861</span><span class="annotate long">line 860 didn't jump to line 861</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">880&#x202F;&#x219B;&#x202F;881</span><span class="annotate long">line 880 didn't jump to line 881</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-1B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">900&#x202F;&#x219B;&#x202F;901</span><span class="annotate long">line 900 didn't jump to line 901</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-3B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">924&#x202F;&#x219B;&#x202F;925</span><span class="annotate long">line 924 didn't jump to line 925</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">24</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">948&#x202F;&#x219B;&#x202F;949</span><span class="annotate long">line 948 didn't jump to line 949</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">972&#x202F;&#x219B;&#x202F;973</span><span class="annotate long">line 972 didn't jump to line 973</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">attention_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">window_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_ctx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">ffn_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">4</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">        <span class="nam">rotary_pct</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_pct</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">rotary_pct</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t728" href="#t728">728</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t729" href="#t729">729</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t730" href="#t730">730</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t731" href="#t731">731</a></span><span class="t"><span class="str">    Returns the model config for a HuggingFace model, converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t732" href="#t732">732</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t733" href="#t733">733</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t734" href="#t734">734</a></span><span class="t"><span class="str">    Takes the official_model_name as an input.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t735" href="#t735">735</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t736" href="#t736">736</a></span><span class="t">    <span class="com"># In case the user passed in an alias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t737" href="#t737">737</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span> <span class="op">/</span> <span class="str">"config.json"</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">737&#x202F;&#x219B;&#x202F;738</span><span class="annotate long">line 737 didn't jump to line 738, because the condition on line 737 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t738" href="#t738">738</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">"Loading model config from local directory"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t739" href="#t739">739</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t740" href="#t740">740</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t741" href="#t741">741</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t742" href="#t742">742</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t743" href="#t743">743</a></span><span class="t">    <span class="com"># Load HuggingFace model config</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t744" href="#t744">744</a></span><span class="t">    <span class="key">if</span> <span class="str">"llama"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">744&#x202F;&#x219B;&#x202F;745</span><span class="annotate long">line 744 didn't jump to line 745, because the condition on line 744 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t745" href="#t745">745</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"LlamaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t746" href="#t746">746</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma-2"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">746&#x202F;&#x219B;&#x202F;747</span><span class="annotate long">line 746 didn't jump to line 747, because the condition on line 746 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t747" href="#t747">747</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"Gemma2ForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t748" href="#t748">748</a></span><span class="t">    <span class="key">elif</span> <span class="str">"gemma"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">748&#x202F;&#x219B;&#x202F;749</span><span class="annotate long">line 748 didn't jump to line 749, because the condition on line 748 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t749" href="#t749">749</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="str">"GemmaForCausalLM"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t750" href="#t750">750</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t751" href="#t751">751</a></span><span class="t">        <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t752" href="#t752">752</a></span><span class="t">        <span class="nam">hf_config</span> <span class="op">=</span> <span class="nam">AutoConfig</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t753" href="#t753">753</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t754" href="#t754">754</a></span><span class="t">            <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t755" href="#t755">755</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t756" href="#t756">756</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t757" href="#t757">757</a></span><span class="t">        <span class="nam">architecture</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">architectures</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t758" href="#t758">758</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t759" href="#t759">759</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">759&#x202F;&#x219B;&#x202F;762</span><span class="annotate long">line 759 didn't jump to line 762</span></span></p>
+    <p class="pln"><span class="n"><a id="t760" href="#t760">760</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-7b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-7b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t761" href="#t761">761</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t762" href="#t762">762</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t763" href="#t763">763</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t764" href="#t764">764</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t765" href="#t765">765</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t766" href="#t766">766</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t767" href="#t767">767</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t768" href="#t768">768</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t769" href="#t769">769</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-7b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t770" href="#t770">770</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t771" href="#t771">771</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t772" href="#t772">772</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t773" href="#t773">773</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t774" href="#t774">774</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t775" href="#t775">775</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t776" href="#t776">776</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t777" href="#t777">777</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t778" href="#t778">778</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t779" href="#t779">779</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"codellama"</span><span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture CodeLlama and Llama-2</span>&nbsp;</span><span class="r"><span class="annotate short">779&#x202F;&#x219B;&#x202F;780</span><span class="annotate long">line 779 didn't jump to line 780</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t780" href="#t780">780</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t781" href="#t781">781</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t782" href="#t782">782</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t783" href="#t783">783</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t784" href="#t784">784</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">11008</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t785" href="#t785">785</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t786" href="#t786">786</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t787" href="#t787">787</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t788" href="#t788">788</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t789" href="#t789">789</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t790" href="#t790">790</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t791" href="#t791">791</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t792" href="#t792">792</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">4096</span> <span class="op">//</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t793" href="#t793">793</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t794" href="#t794">794</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t795" href="#t795">795</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">1000000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t796" href="#t796">796</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t797" href="#t797">797</a></span><span class="t">        <span class="key">if</span> <span class="str">"python"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t798" href="#t798">798</a></span><span class="t">            <span class="com"># The vocab size of python version of CodeLlama-7b is 32000</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t799" href="#t799">799</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span> <span class="op">=</span> <span class="num">32000</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t800" href="#t800">800</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">800&#x202F;&#x219B;&#x202F;803</span><span class="annotate long">line 800 didn't jump to line 803</span></span></p>
+    <p class="pln"><span class="n"><a id="t801" href="#t801">801</a></span><span class="t">        <span class="op">(</span><span class="str">"llama-13b"</span><span class="op">,</span> <span class="str">"meta-llama/Llama-2-13b"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t802" href="#t802">802</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>  <span class="com"># same architecture for LLaMA and Llama-2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t803" href="#t803">803</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t804" href="#t804">804</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">5120</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t805" href="#t805">805</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t806" href="#t806">806</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t807" href="#t807">807</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">13824</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t808" href="#t808">808</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t809" href="#t809">809</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t810" href="#t810">810</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span> <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"llama-13b"</span><span class="op">)</span> <span class="key">else</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t811" href="#t811">811</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t812" href="#t812">812</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t813" href="#t813">813</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t814" href="#t814">814</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t815" href="#t815">815</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t816" href="#t816">816</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">5120</span> <span class="op">//</span> <span class="num">40</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t817" href="#t817">817</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t818" href="#t818">818</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t819" href="#t819">819</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t820" href="#t820">820</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-30b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">820&#x202F;&#x219B;&#x202F;821</span><span class="annotate long">line 820 didn't jump to line 821</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t821" href="#t821">821</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t822" href="#t822">822</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">6656</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t823" href="#t823">823</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t824" href="#t824">824</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t825" href="#t825">825</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">17920</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t826" href="#t826">826</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">60</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t827" href="#t827">827</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t828" href="#t828">828</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t829" href="#t829">829</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t830" href="#t830">830</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t831" href="#t831">831</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t832" href="#t832">832</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t833" href="#t833">833</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t834" href="#t834">834</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">6656</span> <span class="op">//</span> <span class="num">52</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t835" href="#t835">835</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t836" href="#t836">836</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t837" href="#t837">837</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t838" href="#t838">838</a></span><span class="t">    <span class="key">elif</span> <span class="str">"llama-65b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">838&#x202F;&#x219B;&#x202F;839</span><span class="annotate long">line 838 didn't jump to line 839</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t839" href="#t839">839</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t840" href="#t840">840</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t841" href="#t841">841</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t842" href="#t842">842</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t843" href="#t843">843</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">22016</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t844" href="#t844">844</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t845" href="#t845">845</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t846" href="#t846">846</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-6</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t847" href="#t847">847</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t848" href="#t848">848</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t849" href="#t849">849</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t850" href="#t850">850</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t851" href="#t851">851</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">8192</span> <span class="op">//</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t852" href="#t852">852</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t853" href="#t853">853</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t854" href="#t854">854</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t855" href="#t855">855</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t856" href="#t856">856</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-2-70b"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">856&#x202F;&#x219B;&#x202F;857</span><span class="annotate long">line 856 didn't jump to line 857</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t857" href="#t857">857</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t858" href="#t858">858</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t859" href="#t859">859</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t860" href="#t860">860</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t861" href="#t861">861</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t862" href="#t862">862</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t863" href="#t863">863</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t864" href="#t864">864</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t865" href="#t865">865</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">32000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t866" href="#t866">866</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t867" href="#t867">867</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t868" href="#t868">868</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t869" href="#t869">869</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t870" href="#t870">870</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t871" href="#t871">871</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t872" href="#t872">872</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t873" href="#t873">873</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t874" href="#t874">874</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t875" href="#t875">875</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">875&#x202F;&#x219B;&#x202F;876</span><span class="annotate long">line 875 didn't jump to line 876</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t876" href="#t876">876</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t877" href="#t877">877</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t878" href="#t878">878</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t879" href="#t879">879</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t880" href="#t880">880</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t881" href="#t881">881</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t882" href="#t882">882</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t883" href="#t883">883</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t884" href="#t884">884</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t885" href="#t885">885</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t886" href="#t886">886</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t887" href="#t887">887</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t888" href="#t888">888</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t889" href="#t889">889</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t890" href="#t890">890</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t891" href="#t891">891</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t892" href="#t892">892</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t893" href="#t893">893</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t894" href="#t894">894</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t895" href="#t895">895</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Meta-Llama-3-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">895&#x202F;&#x219B;&#x202F;896</span><span class="annotate long">line 895 didn't jump to line 896</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t896" href="#t896">896</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t897" href="#t897">897</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t898" href="#t898">898</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t899" href="#t899">899</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t900" href="#t900">900</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t901" href="#t901">901</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t902" href="#t902">902</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t903" href="#t903">903</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t904" href="#t904">904</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t905" href="#t905">905</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t906" href="#t906">906</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t907" href="#t907">907</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t908" href="#t908">908</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t909" href="#t909">909</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t910" href="#t910">910</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t911" href="#t911">911</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t912" href="#t912">912</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t913" href="#t913">913</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t914" href="#t914">914</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t915" href="#t915">915</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-1B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">915&#x202F;&#x219B;&#x202F;916</span><span class="annotate long">line 915 didn't jump to line 916</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t916" href="#t916">916</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t917" href="#t917">917</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t918" href="#t918">918</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t919" href="#t919">919</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t920" href="#t920">920</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t921" href="#t921">921</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t922" href="#t922">922</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t923" href="#t923">923</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t924" href="#t924">924</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t925" href="#t925">925</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t926" href="#t926">926</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t927" href="#t927">927</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t928" href="#t928">928</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t929" href="#t929">929</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t930" href="#t930">930</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t931" href="#t931">931</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t932" href="#t932">932</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t933" href="#t933">933</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t934" href="#t934">934</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t935" href="#t935">935</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t936" href="#t936">936</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t937" href="#t937">937</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t938" href="#t938">938</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t939" href="#t939">939</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.2-3B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">939&#x202F;&#x219B;&#x202F;940</span><span class="annotate long">line 939 didn't jump to line 940</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t940" href="#t940">940</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t941" href="#t941">941</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t942" href="#t942">942</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t943" href="#t943">943</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">24</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t944" href="#t944">944</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t945" href="#t945">945</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t946" href="#t946">946</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t947" href="#t947">947</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t948" href="#t948">948</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t949" href="#t949">949</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t950" href="#t950">950</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t951" href="#t951">951</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t952" href="#t952">952</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t953" href="#t953">953</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t954" href="#t954">954</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t955" href="#t955">955</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t956" href="#t956">956</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t957" href="#t957">957</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t958" href="#t958">958</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t959" href="#t959">959</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t960" href="#t960">960</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t961" href="#t961">961</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">32.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t962" href="#t962">962</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t963" href="#t963">963</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-8B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">963&#x202F;&#x219B;&#x202F;964</span><span class="annotate long">line 963 didn't jump to line 964</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t964" href="#t964">964</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t965" href="#t965">965</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t966" href="#t966">966</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t967" href="#t967">967</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t968" href="#t968">968</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t969" href="#t969">969</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t970" href="#t970">970</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t971" href="#t971">971</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t972" href="#t972">972</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t973" href="#t973">973</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t974" href="#t974">974</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t975" href="#t975">975</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t976" href="#t976">976</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t977" href="#t977">977</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t978" href="#t978">978</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t979" href="#t979">979</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t980" href="#t980">980</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t981" href="#t981">981</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t982" href="#t982">982</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t983" href="#t983">983</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t984" href="#t984">984</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t985" href="#t985">985</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t986" href="#t986">986</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t987" href="#t987">987</a></span><span class="t">    <span class="key">elif</span> <span class="str">"Llama-3.1-70B"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">987&#x202F;&#x219B;&#x202F;988</span><span class="annotate long">line 987 didn't jump to line 988</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t988" href="#t988">988</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t989" href="#t989">989</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t990" href="#t990">990</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t991" href="#t991">991</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">64</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t992" href="#t992">992</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">28672</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t993" href="#t993">993</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">80</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t994" href="#t994">994</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t995" href="#t995">995</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t996" href="#t996">996</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">128256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t997" href="#t997">997</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t998" href="#t998">998</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t999" href="#t999">999</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1000" href="#t1000">1000</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1001" href="#t1001">1001</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1002" href="#t1002">1002</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1003" href="#t1003">1003</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1004" href="#t1004">1004</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1005" href="#t1005">1005</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">500000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1006" href="#t1006">1006</a></span><span class="t">            <span class="str">"use_NTK_by_parts_rope"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1007" href="#t1007">1007</a></span><span class="t">            <span class="str">"NTK_by_parts_low_freq_factor"</span><span class="op">:</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1008" href="#t1008">1008</a></span><span class="t">            <span class="str">"NTK_by_parts_high_freq_factor"</span><span class="op">:</span> <span class="num">4.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1009" href="#t1009">1009</a></span><span class="t">            <span class="str">"NTK_by_parts_factor"</span><span class="op">:</span> <span class="num">8.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1010" href="#t1010">1010</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1011" href="#t1011">1011</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1012" href="#t1012">1012</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1013" href="#t1013">1013</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1014" href="#t1014">1014</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1015" href="#t1015">1015</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1016" href="#t1016">1016</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1017" href="#t1017">1017</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1018" href="#t1018">1018</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1019" href="#t1019">1019</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1020" href="#t1020">1020</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1021" href="#t1021">1021</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">attention_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1022" href="#t1022">1022</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1023" href="#t1023">1023</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1024" href="#t1024">1024</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1025" href="#t1025">1025</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">window_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1026" href="#t1026">1026</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1027" href="#t1027">1027</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1028" href="#t1028">1028</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1029" href="#t1029">1029</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1030" href="#t1030">1030</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1031" href="#t1031">1031</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1032" href="#t1032">1032</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1033" href="#t1033">1033</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1034" href="#t1034">1034</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1035" href="#t1035">1035</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1036" href="#t1036">1036</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_ctx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1037" href="#t1037">1037</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1038" href="#t1038">1038</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1039" href="#t1039">1039</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1040" href="#t1040">1040</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1041" href="#t1041">1041</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1042" href="#t1042">1042</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1043" href="#t1043">1043</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1044" href="#t1044">1044</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1045" href="#t1045">1045</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1046" href="#t1046">1046</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1047" href="#t1047">1047</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1048" href="#t1048">1048</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1049" href="#t1049">1049</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1050" href="#t1050">1050</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">ffn_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1051" href="#t1051">1051</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1052" href="#t1052">1052</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1053" href="#t1053">1053</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1054" href="#t1054">1054</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1055" href="#t1055">1055</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1056" href="#t1056">1056</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1057" href="#t1057">1057</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1058" href="#t1058">1058</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1059" href="#t1059">1059</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1060" href="#t1060">1060</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1061" href="#t1061">1061</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1062" href="#t1062">1062</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1063" href="#t1063">1063</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1064" href="#t1064">1064</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1065" href="#t1065">1065</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1066" href="#t1066">1066</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">4</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1067" href="#t1067">1067</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1068" href="#t1068">1068</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1069" href="#t1069">1069</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-5</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1100&#x202F;&#x219B;&#x202F;1101</span><span class="annotate long">line 1100 didn't jump to line 1101, because the condition on line 1100 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">        <span class="nam">use_local_attn</span> <span class="op">=</span> <span class="key">True</span> <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span> <span class="key">else</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">            <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">hf_config</span><span class="op">,</span> <span class="str">"head_dim"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t">            <span class="key">else</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t">        <span class="nam">rotary_pct</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rotary_pct</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">rotary_pct</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># None if no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span> <span class="key">if</span> <span class="nam">use_local_attn</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="nam">use_local_attn</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1123&#x202F;&#x219B;&#x202F;1124</span><span class="annotate long">line 1123 didn't jump to line 1124</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t">            <span class="str">"dtype"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1115&#x202F;&#x219B;&#x202F;1116</span><span class="annotate long">line 1115 didn't jump to line 1116, because the condition on line 1115 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t">        <span class="nam">use_local_attn</span> <span class="op">=</span> <span class="key">True</span> <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span> <span class="key">else</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t">            <span class="key">if</span> <span class="nam">hasattr</span><span class="op">(</span><span class="nam">hf_config</span><span class="op">,</span> <span class="str">"head_dim"</span><span class="op">)</span> <span class="key">and</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">head_dim</span> <span class="op">></span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t">            <span class="key">else</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># None if no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span> <span class="key">if</span> <span class="nam">use_local_attn</span> <span class="key">else</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="nam">use_local_attn</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># This is None, as no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">]</span> <span class="op">*</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">            <span class="str">"num_experts"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_local_experts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">            <span class="str">"experts_per_token"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_experts_per_tok</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to HF Tokenizer Constraints</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_fast"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t">            <span class="str">"post_embedding_ln"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"alibi"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1138&#x202F;&#x219B;&#x202F;1139</span><span class="annotate long">line 1138 didn't jump to line 1139</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t">            <span class="str">"dtype"</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">bfloat16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>  <span class="com"># Capped due to memory issues</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">sliding_window</span><span class="op">,</span>  <span class="com"># This is None, as no sliding window was used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">]</span> <span class="op">*</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t">            <span class="str">"num_experts"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_local_experts</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t">            <span class="str">"experts_per_token"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_experts_per_tok</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1162&#x202F;&#x219B;&#x202F;1164</span><span class="annotate long">line 1162 didn't jump to line 1164</span></span></p>
-    <p class="pln"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t">        <span class="com"># santacoder</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="str">"santacoder"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t">            <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">,</span>  <span class="com"># Only santacoder needs trust_remote_code</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1181&#x202F;&#x219B;&#x202F;1182</span><span class="annotate long">line 1181 didn't jump to line 1182</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t">                <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t">                <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span> <span class="op">!=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t">                <span class="key">else</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t">            <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t">            <span class="com"># This is done because the current implementation of GQA will use Grouped-Query Attention if</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t">            <span class="com"># n_key_value_heads is not None, but hf_config.num_key_value_heads is sometimes specified as</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t">            <span class="com"># the same as hf_config.num_attention_heads, in which case GQA should not be used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1207&#x202F;&#x219B;&#x202F;1208</span><span class="annotate long">line 1207 didn't jump to line 1208</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">kv_channels</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1229&#x202F;&#x219B;&#x202F;1231</span><span class="annotate long">line 1229 didn't jump to line 1231</span></span></p>
-    <p class="pln"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">        <span class="com"># Note that Qwen1.5 models have architecture type Qwen2ForCausalLM.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1253&#x202F;&#x219B;&#x202F;1255</span><span class="annotate long">line 1253 didn't jump to line 1255</span></span></p>
-    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">        <span class="nam">partial_rotary_factor</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">partial_rotary_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">partial_rotary_factor</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1275&#x202F;&#x219B;&#x202F;1277</span><span class="annotate long">line 1275 didn't jump to line 1277</span></span></p>
-    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi3 models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1298&#x202F;&#x219B;&#x202F;1300</span><span class="annotate long">line 1298 didn't jump to line 1300</span></span></p>
-    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">        <span class="com"># Architecture for Gemma 2b and Gemma 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">16384</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">18</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-7b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1320&#x202F;&#x219B;&#x202F;1322</span><span class="annotate long">line 1320 didn't jump to line 1322</span></span></p>
-    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">        <span class="com"># Architecture for Gemma 7b and Gemma 7b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">24576</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1342&#x202F;&#x219B;&#x202F;1344</span><span class="annotate long">line 1342 didn't jump to line 1344</span></span></p>
-    <p class="pln"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 2b and Gemma-2 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2304</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">9216</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">26</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-9b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1369&#x202F;&#x219B;&#x202F;1371</span><span class="annotate long">line 1369 didn't jump to line 1371</span></span></p>
-    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 9b and Gemma-2 9b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3584</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">42</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-27b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1396&#x202F;&#x219B;&#x202F;1398</span><span class="annotate long">line 1396 didn't jump to line 1398</span></span></p>
-    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 27b and Gemma-2 27b Instruct models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4608</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">36864</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">46</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">            <span class="str">"attn_scale"</span><span class="op">:</span> <span class="num">12.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">23</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1424&#x202F;&#x219B;&#x202F;1444</span><span class="annotate long">line 1424 didn't jump to line 1444, because the condition on line 1424 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_kv</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_ff</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">feed_forward_proj</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"relative_positional_bias"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">            <span class="str">"relative_attention_max_distance"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_max_distance</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">            <span class="str">"relative_attention_num_buckets"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_num_buckets</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">            <span class="str">"decoder_start_token_id"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">decoder_start_token_id</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">            <span class="str">"tie_word_embeddings"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">tie_word_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">        <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">f"{architecture} is not currently supported."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">    <span class="com"># All of these models use LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"original_architecture"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">architecture</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">    <span class="com"># The name such that AutoTokenizer.from_pretrained works</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"tokenizer_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">    <span class="key">if</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1449&#x202F;&#x219B;&#x202F;1450</span><span class="annotate long">line 1449 didn't jump to line 1450, because the condition on line 1449 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t"><span class="str">    Loads the config for a model trained by me (NeelNanda), converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t"><span class="str">    AutoConfig is not supported, because these models are in the HookedTransformer format, so we directly download and load the json.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">    <span class="nam">cfg_json</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="str">"config.json"</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">    <span class="nam">cfg_arch</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">        <span class="str">"architecture"</span><span class="op">,</span> <span class="str">"neel"</span> <span class="key">if</span> <span class="str">"_old"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">official_model_name</span> <span class="key">else</span> <span class="str">"neel-solu-old"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">    <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">        <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">        <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">        <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_mlp"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">        <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">        <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_heads"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">        <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_ctx"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">        <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">        <span class="str">"tokenizer_name"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"tokenizer_name"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">        <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"act_fn"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">        <span class="str">"attn_only"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"attn_only"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">        <span class="str">"final_rms"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"final_rms"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">        <span class="str">"original_architecture"</span><span class="op">:</span> <span class="nam">cfg_arch</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">    <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">    <span class="key">if</span> <span class="str">"normalization"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">    <span class="key">if</span> <span class="str">"shortformer_pos"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">            <span class="str">"shortformer"</span> <span class="key">if</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"shortformer_pos"</span><span class="op">]</span> <span class="key">else</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">    <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">    <span class="nam">hf_cfg</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">    <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">    <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">    <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">    <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">    <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">    <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">    <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped due to HF Tokenizer Constraints</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_fast"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t">            <span class="str">"post_embedding_ln"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"alibi"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t">            <span class="str">"default_prepend_bos"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1178&#x202F;&#x219B;&#x202F;1180</span><span class="annotate long">line 1178 didn't jump to line 1180</span></span></p>
+    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t">        <span class="com"># santacoder</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_embd</span> <span class="op">*</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_layer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">n_positions</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">activation_function</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="str">"santacoder"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t">            <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">,</span>  <span class="com"># Only santacoder needs trust_remote_code</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t">            <span class="str">"scale_attn_by_inverse_layer_idx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_by_inverse_layer_idx</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1197&#x202F;&#x219B;&#x202F;1198</span><span class="annotate long">line 1197 didn't jump to line 1198</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t">                <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">                <span class="key">if</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span> <span class="op">!=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">                <span class="key">else</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">            <span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">            <span class="com"># This is done because the current implementation of GQA will use Grouped-Query Attention if</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">            <span class="com"># n_key_value_heads is not None, but hf_config.num_key_value_heads is sometimes specified as</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">            <span class="com"># the same as hf_config.num_attention_heads, in which case GQA should not be used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1223&#x202F;&#x219B;&#x202F;1224</span><span class="annotate long">line 1223 didn't jump to line 1224</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"silu"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">scale_attn_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">kv_channels</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1245&#x202F;&#x219B;&#x202F;1247</span><span class="annotate long">line 1245 didn't jump to line 1247</span></span></p>
+    <p class="pln"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">        <span class="com"># Note that Qwen1.5 models have architecture type Qwen2ForCausalLM.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_key_value_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>  <span class="com"># Capped bc the actual ctx length is 30k and the attn mask would be too big</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="str">"rotary_adjacent_pairs"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="str">"tokenizer_prepends_bos"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1269&#x202F;&#x219B;&#x202F;1271</span><span class="annotate long">line 1269 didn't jump to line 1271</span></span></p>
+    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"LN"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">        <span class="nam">partial_rotary_factor</span> <span class="op">=</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">partial_rotary_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"rotary_dim"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">round</span><span class="op">(</span><span class="nam">partial_rotary_factor</span> <span class="op">*</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1291&#x202F;&#x219B;&#x202F;1293</span><span class="annotate long">line 1291 didn't jump to line 1293</span></span></p>
+    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">        <span class="com"># Architecture for microsoft/phi3 models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">intermediate_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_hidden_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_position_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rms_norm_eps</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_act</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">rope_theta</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">            <span class="str">"parallel_attn_mlp"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">hidden_size</span> <span class="op">//</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_attention_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1314&#x202F;&#x219B;&#x202F;1316</span><span class="annotate long">line 1314 didn't jump to line 1316</span></span></p>
+    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">        <span class="com"># Architecture for Gemma 2b and Gemma 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2048</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">16384</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">18</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-7b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1336&#x202F;&#x219B;&#x202F;1338</span><span class="annotate long">line 1336 didn't jump to line 1338</span></span></p>
+    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">        <span class="com"># Architecture for Gemma 7b and Gemma 7b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3072</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">24576</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">28</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_new"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="str">"rotary_dim"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-2b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1358&#x202F;&#x219B;&#x202F;1360</span><span class="annotate long">line 1358 didn't jump to line 1360</span></span></p>
+    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 2b and Gemma-2 2b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">2304</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">9216</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">26</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">4</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-9b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1385&#x202F;&#x219B;&#x202F;1387</span><span class="annotate long">line 1385 didn't jump to line 1387</span></span></p>
+    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 9b and Gemma-2 9b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">3584</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">256</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">14336</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">42</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">8</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">21</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"google/gemma-2-27b"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1412&#x202F;&#x219B;&#x202F;1414</span><span class="annotate long">line 1412 didn't jump to line 1414</span></span></p>
+    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">        <span class="com"># Architecture for Gemma-2 27b and Gemma-2 27b Instruct models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="num">4608</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="num">128</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="num">32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="num">36864</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="num">46</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="num">8192</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="num">1e-06</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="num">256000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="str">"gelu_pytorch_tanh"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t">            <span class="str">"initializer_range"</span><span class="op">:</span> <span class="num">0.02</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t">            <span class="str">"normalization_type"</span><span class="op">:</span> <span class="str">"RMS"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">            <span class="str">"rotary_base"</span><span class="op">:</span> <span class="num">10000.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"rotary"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">            <span class="str">"attn_scale"</span><span class="op">:</span> <span class="num">12.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">            <span class="str">"n_key_value_heads"</span><span class="op">:</span> <span class="num">16</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">            <span class="str">"window_size"</span><span class="op">:</span> <span class="num">4096</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">            <span class="str">"use_local_attn"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">            <span class="str">"attn_types"</span><span class="op">:</span> <span class="op">[</span><span class="str">"global"</span><span class="op">,</span> <span class="str">"local"</span><span class="op">]</span> <span class="op">*</span> <span class="num">23</span><span class="op">,</span>  <span class="com"># Alternate global and local attn</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">            <span class="str">"attn_scores_soft_cap"</span><span class="op">:</span> <span class="num">50.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">            <span class="str">"output_logits_soft_cap"</span><span class="op">:</span> <span class="num">30.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">            <span class="str">"gated_mlp"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">            <span class="str">"final_rms"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">            <span class="str">"use_normalization_before_and_after"</span><span class="op">:</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">    <span class="key">elif</span> <span class="nam">architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1440&#x202F;&#x219B;&#x202F;1460</span><span class="annotate long">line 1440 didn't jump to line 1460, because the condition on line 1440 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">            <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">            <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_kv</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">            <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">            <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">d_ff</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">            <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">vocab_size</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">            <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">num_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">            <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">max_length</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">            <span class="str">"eps"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">layer_norm_epsilon</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">            <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">feed_forward_proj</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">            <span class="str">"positional_embedding_type"</span><span class="op">:</span> <span class="str">"relative_positional_bias"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">            <span class="str">"relative_attention_max_distance"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_max_distance</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">            <span class="str">"relative_attention_num_buckets"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">relative_attention_num_buckets</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">            <span class="str">"decoder_start_token_id"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">decoder_start_token_id</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">            <span class="str">"attention_dir"</span><span class="op">:</span> <span class="str">"bidirectional"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">            <span class="str">"use_attn_scale"</span><span class="op">:</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">            <span class="str">"tie_word_embeddings"</span><span class="op">:</span> <span class="nam">hf_config</span><span class="op">.</span><span class="nam">tie_word_embeddings</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">        <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">f"{architecture} is not currently supported."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">    <span class="com"># All of these models use LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"original_architecture"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">architecture</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">    <span class="com"># The name such that AutoTokenizer.from_pretrained works</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"tokenizer_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">    <span class="key">if</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1465&#x202F;&#x219B;&#x202F;1466</span><span class="annotate long">line 1465 didn't jump to line 1466, because the condition on line 1465 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t"><span class="key">def</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t"><span class="str">    Loads the config for a model trained by me (NeelNanda), converted to a dictionary</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t"><span class="str">    in the HookedTransformerConfig format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t"><span class="str">    AutoConfig is not supported, because these models are in the HookedTransformer format, so we directly download and load the json.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">    <span class="nam">cfg_json</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="str">"config.json"</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">    <span class="nam">cfg_arch</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">        <span class="str">"architecture"</span><span class="op">,</span> <span class="str">"neel"</span> <span class="key">if</span> <span class="str">"_old"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">official_model_name</span> <span class="key">else</span> <span class="str">"neel-solu-old"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">    <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">        <span class="str">"d_model"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">        <span class="str">"n_layers"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="str">"d_mlp"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_mlp"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="str">"d_head"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="str">"n_heads"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_heads"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">        <span class="str">"n_ctx"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"n_ctx"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">        <span class="str">"d_vocab"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"d_vocab"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">        <span class="str">"tokenizer_name"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"tokenizer_name"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">        <span class="str">"act_fn"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"act_fn"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">        <span class="str">"attn_only"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"attn_only"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">        <span class="str">"final_rms"</span><span class="op">:</span> <span class="nam">cfg_json</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"final_rms"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">        <span class="str">"original_architecture"</span><span class="op">:</span> <span class="nam">cfg_arch</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">    <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">    <span class="key">if</span> <span class="str">"normalization"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">    <span class="key">if</span> <span class="str">"shortformer_pos"</span> <span class="key">in</span> <span class="nam">cfg_json</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">            <span class="str">"shortformer"</span> <span class="key">if</span> <span class="nam">cfg_json</span><span class="op">[</span><span class="str">"shortformer_pos"</span><span class="op">]</span> <span class="key">else</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"standard"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="str">            Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="str">            1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="str">            2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="str">            3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">            Even for models not explicitly trained with the BOS token, heads often use the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t"><span class="str">            so this empirically seems to give better results. Note that you can also locally override the default behavior</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">            by passing in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1546&#x202F;&#x219B;&#x202F;1548</span><span class="annotate long">line 1546 didn't jump to line 1548, because the condition on line 1546 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1559&#x202F;&#x219B;&#x202F;1562</span><span class="annotate long">line 1559 didn't jump to line 1562, because the condition on line 1559 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1589&#x202F;&#x219B;&#x202F;1591</span><span class="annotate long">line 1589 didn't jump to line 1591, because the condition on line 1589 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1596&#x202F;&#x219B;&#x202F;1597</span><span class="annotate long">line 1596 didn't jump to line 1597, because the condition on line 1596 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">    <span class="key">if</span> <span class="nam">default_prepend_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">        <span class="com"># User explicitly set prepend_bos behavior, override config/default value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">    <span class="key">elif</span> <span class="str">"default_prepend_bos"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">cfg_dict</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1621&#x202F;&#x219B;&#x202F;1625</span><span class="annotate long">line 1621 didn't jump to line 1625, because the condition on line 1621 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="com"># No config value or user override, set default value (True)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">    <span class="key">if</span> <span class="nam">first_n_layers</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1627&#x202F;&#x219B;&#x202F;1628</span><span class="annotate long">line 1627 didn't jump to line 1628, because the condition on line 1627 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">first_n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">    <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">    <span class="nam">hf_cfg</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">dict</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t">    <span class="nam">checkpoint_index</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">    <span class="nam">checkpoint_value</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">    <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">    <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">    <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">    <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">    <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t"><span class="str">            Resolution order for default_prepend_bos:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t"><span class="str">            1. If user passes value explicitly, use that value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t"><span class="str">            2. Model-specific default from cfg_dict if it exists (e.g. for bloom models it's False)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t"><span class="str">            3. Global default (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t"><span class="str">            Even for models not explicitly trained with the BOS token, heads often use the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t"><span class="str">            so this empirically seems to give better results. Note that you can also locally override the default behavior</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t"><span class="str">            by passing in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1562&#x202F;&#x219B;&#x202F;1564</span><span class="annotate long">line 1562 didn't jump to line 1564, because the condition on line 1562 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1575&#x202F;&#x219B;&#x202F;1578</span><span class="annotate long">line 1575 didn't jump to line 1578, because the condition on line 1575 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1605&#x202F;&#x219B;&#x202F;1607</span><span class="annotate long">line 1605 didn't jump to line 1607, because the condition on line 1605 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1612&#x202F;&#x219B;&#x202F;1613</span><span class="annotate long">line 1612 didn't jump to line 1613, because the condition on line 1612 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">    <span class="key">if</span> <span class="nam">default_prepend_bos</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">        <span class="com"># User explicitly set prepend_bos behavior, override config/default value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">    <span class="key">elif</span> <span class="str">"default_prepend_bos"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">cfg_dict</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">        <span class="com"># No config value or user override, set default value (True)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1713&#x202F;&#x219B;&#x202F;1714</span><span class="annotate long">line 1713 didn't jump to line 1714, because the condition on line 1713 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1716&#x202F;&#x219B;&#x202F;1717</span><span class="annotate long">line 1716 didn't jump to line 1717, because the condition on line 1716 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1721&#x202F;&#x219B;&#x202F;1724</span><span class="annotate long">line 1721 didn't jump to line 1724, because the condition on line 1721 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1738&#x202F;&#x219B;&#x202F;1739</span><span class="annotate long">line 1738 didn't jump to line 1739, because the condition on line 1738 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1755&#x202F;&#x219B;&#x202F;1756</span><span class="annotate long">line 1755 didn't jump to line 1756, because the condition on line 1755 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1775&#x202F;&#x219B;&#x202F;1803</span><span class="annotate long">line 1775 didn't jump to line 1803, because the condition on line 1775 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1777&#x202F;&#x219B;&#x202F;1778</span><span class="annotate long">line 1777 didn't jump to line 1778, because the condition on line 1777 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1812&#x202F;&#x219B;&#x202F;1813</span><span class="annotate long">line 1812 didn't jump to line 1813, because the condition on line 1812 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1816&#x202F;&#x219B;&#x202F;1817</span><span class="annotate long">line 1816 didn't jump to line 1817, because the condition on line 1816 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1822&#x202F;&#x219B;&#x202F;1823</span><span class="annotate long">line 1822 didn't jump to line 1823, because the condition on line 1822 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1824&#x202F;&#x219B;&#x202F;1825</span><span class="annotate long">line 1824 didn't jump to line 1825, because the condition on line 1824 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1826&#x202F;&#x219B;&#x202F;1828</span><span class="annotate long">line 1826 didn't jump to line 1828, because the condition on line 1826 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1867&#x202F;&#x219B;&#x202F;1869</span><span class="annotate long">line 1867 didn't jump to line 1869, because the condition on line 1867 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1880&#x202F;&#x219B;&#x202F;1882</span><span class="annotate long">line 1880 didn't jump to line 1882, because </span></span></p>
-    <p class="run"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">    <span class="key">if</span> <span class="nam">first_n_layers</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1643&#x202F;&#x219B;&#x202F;1644</span><span class="annotate long">line 1643 didn't jump to line 1644, because the condition on line 1643 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">first_n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1729&#x202F;&#x219B;&#x202F;1730</span><span class="annotate long">line 1729 didn't jump to line 1730, because the condition on line 1729 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1732&#x202F;&#x219B;&#x202F;1733</span><span class="annotate long">line 1732 didn't jump to line 1733, because the condition on line 1732 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1737&#x202F;&#x219B;&#x202F;1740</span><span class="annotate long">line 1737 didn't jump to line 1740, because the condition on line 1737 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1754&#x202F;&#x219B;&#x202F;1755</span><span class="annotate long">line 1754 didn't jump to line 1755, because the condition on line 1754 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1771&#x202F;&#x219B;&#x202F;1772</span><span class="annotate long">line 1771 didn't jump to line 1772, because the condition on line 1771 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1791&#x202F;&#x219B;&#x202F;1819</span><span class="annotate long">line 1791 didn't jump to line 1819, because the condition on line 1791 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1793&#x202F;&#x219B;&#x202F;1794</span><span class="annotate long">line 1793 didn't jump to line 1794, because the condition on line 1793 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1828&#x202F;&#x219B;&#x202F;1829</span><span class="annotate long">line 1828 didn't jump to line 1829, because the condition on line 1828 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1832&#x202F;&#x219B;&#x202F;1833</span><span class="annotate long">line 1832 didn't jump to line 1833, because the condition on line 1832 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1838&#x202F;&#x219B;&#x202F;1839</span><span class="annotate long">line 1838 didn't jump to line 1839, because the condition on line 1838 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1840&#x202F;&#x219B;&#x202F;1841</span><span class="annotate long">line 1840 didn't jump to line 1841, because the condition on line 1840 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1842&#x202F;&#x219B;&#x202F;1844</span><span class="annotate long">line 1842 didn't jump to line 1844, because the condition on line 1842 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1883&#x202F;&#x219B;&#x202F;1885</span><span class="annotate long">line 1883 didn't jump to line 1885, because the condition on line 1883 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1896&#x202F;&#x219B;&#x202F;1898</span><span class="annotate long">line 1896 didn't jump to line 1898, because </span></span></p>
+    <p class="run"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -2005,7 +2021,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
index 42d4fedab..3cc8b85b5 100644
--- a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -206,7 +206,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_patching_py.html b/_static/coverage/d_af97b5493da09a14_patching_py.html
index 9d3202324..a2652d699 100644
--- a/_static/coverage/d_af97b5493da09a14_patching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_patching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -833,7 +833,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_train_py.html b/_static/coverage/d_af97b5493da09a14_train_py.html
index 9c2046262..2604e2873 100644
--- a/_static/coverage/d_af97b5493da09a14_train_py.html
+++ b/_static/coverage/d_af97b5493da09a14_train_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -245,7 +245,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_utils_py.html b/_static/coverage/d_af97b5493da09a14_utils_py.html
index 5ff590726..0c212a1c4 100644
--- a/_static/coverage/d_af97b5493da09a14_utils_py.html
+++ b/_static/coverage/d_af97b5493da09a14_utils_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1381,7 +1381,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7___init___py.html b/_static/coverage/d_b2114f845e0399b7___init___py.html
index e29cc02bd..00b8e82bb 100644
--- a/_static/coverage/d_b2114f845e0399b7___init___py.html
+++ b/_static/coverage/d_b2114f845e0399b7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
index fd660567c..05cc5cc0f 100644
--- a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -117,7 +117,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_addmm_py.html b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
index 941b76cdb..8802ddd1a 100644
--- a/_static/coverage/d_b2114f845e0399b7_addmm_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_attention_py.html b/_static/coverage/d_b2114f845e0399b7_attention_py.html
index f91e227ae..9594fdaad 100644
--- a/_static/coverage/d_b2114f845e0399b7_attention_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_attention_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">10 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">10<span class="text"> run</span></button>
+            <span class="text">14 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">14<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">0<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">0<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -86,40 +86,44 @@ <h2>
     <p class="pln"><span class="n"><a id="t2" href="#t2">2</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t3" href="#t3">3</a></span><span class="t"><span class="str">Utilities for attention components.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t4" href="#t4">4</a></span><span class="t"><span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t"><span class="key">import</span> <span class="nam">einops</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span><span class="op">.</span><span class="nam">functional</span> <span class="key">as</span> <span class="nam">F</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t"><span class="key">import</span> <span class="nam">einops</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span><span class="op">.</span><span class="nam">functional</span> <span class="key">as</span> <span class="nam">F</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">def</span> <span class="nam">simple_attn_linear</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t">    <span class="nam">input</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">    <span class="nam">w</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_model d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t">    <span class="nam">b</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t">    <span class="str">"""Linear layer for attention calculation."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">    <span class="nam">w</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">w</span><span class="op">,</span> <span class="str">"head_index d_model d_head -> (head_index d_head) d_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">    <span class="nam">b_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">b</span><span class="op">,</span> <span class="str">"head_index d_head -> (head_index d_head)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t">    <span class="key">return</span> <span class="nam">F</span><span class="op">.</span><span class="nam">linear</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">w</span><span class="op">,</span> <span class="nam">b_</span><span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span><span class="nam">input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span> <span class="nam">b</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">b</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t"><span class="key">def</span> <span class="nam">simple_attn_linear</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">    <span class="nam">input</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t">    <span class="nam">w</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_model d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t">    <span class="nam">b</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">    <span class="str">"""Linear layer for attention calculation."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">    <span class="nam">w</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">w</span><span class="op">,</span> <span class="str">"head_index d_model d_head -> (head_index d_head) d_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t">    <span class="nam">b_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">b</span><span class="op">,</span> <span class="str">"head_index d_head -> (head_index d_head)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">    <span class="key">return</span> <span class="nam">F</span><span class="op">.</span><span class="nam">linear</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">w</span><span class="op">,</span> <span class="nam">b_</span><span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span><span class="nam">input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span> <span class="nam">b</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span> <span class="nam">b</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t"><span class="key">def</span> <span class="nam">complex_attn_linear</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t">    <span class="nam">input</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">    <span class="nam">w</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_model d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">    <span class="nam">b</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">    <span class="str">"""Linear layer for attention calculation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t"><span class="str">    This is almost the same as simple_attn_linear, but the input tensor has an extra head_index dimension, used when calculating the input of each attention head separately.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">    <span class="key">return</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">        <span class="nam">einops</span><span class="op">.</span><span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">            <span class="nam">input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">            <span class="nam">w</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">            <span class="str">"batch pos head_index d_model, head_index d_model d_head -> batch pos head_index d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">        <span class="op">+</span> <span class="nam">b</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"><span class="key">def</span> <span class="nam">complex_attn_linear</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">    <span class="nam">input</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t">    <span class="nam">w</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_model d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t">    <span class="nam">b</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">    <span class="str">"""Linear layer for attention calculation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="str">    This is almost the same as simple_attn_linear, but the input tensor has an extra head_index dimension, used when calculating the input of each attention head separately.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">    <span class="com"># Add singleton dimensions for broadcasting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">    <span class="nam">input</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">        <span class="nam">input</span><span class="op">,</span> <span class="str">"batch pos head_index d_model -> batch pos head_index d_model 1"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">    <span class="nam">w</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">w</span><span class="op">,</span> <span class="str">"head_index d_model d_head -> 1 1 head_index d_model d_head"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">    <span class="com"># Element-wise multiplication and sum over the d_model dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t">    <span class="nam">result</span> <span class="op">=</span> <span class="nam">input</span> <span class="op">*</span> <span class="nam">w</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t">    <span class="nam">result</span> <span class="op">=</span> <span class="nam">result</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t">    <span class="key">return</span> <span class="nam">result</span> <span class="op">+</span> <span class="nam">b</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -129,7 +133,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_devices_py.html b/_static/coverage/d_b2114f845e0399b7_devices_py.html
index 68e2f5afd..9192f259d 100644
--- a/_static/coverage/d_b2114f845e0399b7_devices_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_devices_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
index 62730cbbb..155f673bd 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -111,7 +111,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
index 412d69912..bd4b43bad 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
index 4f8b5b741..8abbc960b 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
index 9844234af..dbafdcf86 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
index d20563cea..9b40eeb20 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -186,7 +186,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
index cdd622cbc..20d7bb9f9 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -151,7 +151,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
index 0b93b0768..d55981ee9 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -141,7 +141,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
index 064f7841f..f87463c3e 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -187,7 +187,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
index 93669e3db..a80e082d2 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
index dd65b6b14..ba347b3c5 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
index dc1a0af45..10e837abf 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
index 1b01be718..af9e32c6b 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -179,7 +179,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
index 6945eb144..d25ae733e 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -129,7 +129,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
index a7e173830..a8325657b 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
index baa62e575..ece3ee9bd 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
index 90aadc20d..ed46de3ba 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -175,7 +175,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
index 0dd6c62c2..93f0f430d 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
index 1033b800c..f2267d8a2 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -155,7 +155,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
index 725ef6786..48dbcd55d 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -167,7 +167,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
index bc4d9abc5..45cde4ff1 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
index a206cff7e..27018b692 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -192,7 +192,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831___init___py.html b/_static/coverage/d_db46118ef83ad831___init___py.html
index ba6dea99c..cd4e69703 100644
--- a/_static/coverage/d_db46118ef83ad831___init___py.html
+++ b/_static/coverage/d_db46118ef83ad831___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -123,7 +123,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
index 3db29f3d0..9fb97ef57 100644
--- a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">236 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">195<span class="text"> run</span></button>
+            <span class="text">240 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">199<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">41<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">19<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -381,403 +381,416 @@ <h2>
     <p class="pln"><span class="n"><a id="t297" href="#t297">297</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t298" href="#t298">298</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t299" href="#t299">299</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">                <span class="nam">w</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">                    <span class="str">"head_index d_head d_model -> d_model head_index d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">                <span class="nam">result</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_result</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">                    <span class="nam">einops</span><span class="op">.</span><span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">                        <span class="nam">z</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">                        <span class="nam">w</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">                        <span class="str">"... head_index d_head, d_model head_index d_head -> ... head_index d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">                <span class="op">)</span>  <span class="com"># [batch, pos, head_index, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">            <span class="nam">out</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">                <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span><span class="nam">result</span><span class="op">,</span> <span class="str">"batch position index model->batch position model"</span><span class="op">,</span> <span class="str">"sum"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">        <span class="key">return</span> <span class="nam">out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_qkv_matrices</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">        <span class="nam">query_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">        <span class="nam">key_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">        <span class="nam">value_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="nam">attn_fn</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">            <span class="nam">complex_attn_linear</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="key">or</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">            <span class="key">else</span> <span class="nam">simple_attn_linear</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">341&#x202F;&#x219B;&#x202F;342</span><span class="annotate long">line 341 didn't jump to line 342, because the condition on line 341 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">            <span class="nam">q</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_q</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">                    <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">                    <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">.</span><span class="nam">quant_state</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_Q</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">            <span class="nam">q</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_q</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">query_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_Q</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">359&#x202F;&#x219B;&#x202F;360</span><span class="annotate long">line 359 didn't jump to line 360, because the condition on line 359 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">,</span> <span class="nam">Params4bit</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"W_K must be a Params4bit object if load_in_4bit is True"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">            <span class="nam">k</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_k</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span> <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">quant_state</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_K</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">            <span class="nam">k</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_k</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">key_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_K</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">377&#x202F;&#x219B;&#x202F;378</span><span class="annotate long">line 377 didn't jump to line 378, because the condition on line 377 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">Params4bit</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"W_V must be a Params4bit object if load_in_4bit is True"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">            <span class="nam">v</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_v</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">                    <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">                    <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">.</span><span class="nam">quant_state</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_V</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">            <span class="nam">v</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_v</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">value_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_V</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">        <span class="key">return</span> <span class="nam">q</span><span class="op">,</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t300" href="#t300">300</a></span><span class="t">                <span class="com"># Add singleton dimensions to make shapes compatible for broadcasting:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t301" href="#t301">301</a></span><span class="t">                <span class="nam">w</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t302" href="#t302">302</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t303" href="#t303">303</a></span><span class="t">                    <span class="str">"head_index d_head d_model -> 1 1 head_index d_head d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t304" href="#t304">304</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t305" href="#t305">305</a></span><span class="t">                <span class="nam">z</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t306" href="#t306">306</a></span><span class="t">                    <span class="nam">z</span><span class="op">,</span> <span class="str">"batch pos head_index d_head -> batch pos head_index d_head 1"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t307" href="#t307">307</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t308" href="#t308">308</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t309" href="#t309">309</a></span><span class="t">                <span class="com"># Multiply the z tensor by the W_O tensor, summing over the d_head dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t310" href="#t310">310</a></span><span class="t">                <span class="nam">unhooked_result</span> <span class="op">=</span> <span class="op">(</span><span class="nam">z</span> <span class="op">*</span> <span class="nam">w</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t311" href="#t311">311</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t312" href="#t312">312</a></span><span class="t">                <span class="nam">result</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_result</span><span class="op">(</span><span class="nam">unhooked_result</span><span class="op">)</span>  <span class="com"># [batch, pos, head_index, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t313" href="#t313">313</a></span><span class="t">            <span class="nam">out</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t314" href="#t314">314</a></span><span class="t">                <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span><span class="nam">result</span><span class="op">,</span> <span class="str">"batch position index model->batch position model"</span><span class="op">,</span> <span class="str">"sum"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t315" href="#t315">315</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t316" href="#t316">316</a></span><span class="t">            <span class="op">)</span>  <span class="com"># [batch, pos, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t317" href="#t317">317</a></span><span class="t">        <span class="key">return</span> <span class="nam">out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t318" href="#t318">318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t319" href="#t319">319</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_qkv_matrices</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t320" href="#t320">320</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t321" href="#t321">321</a></span><span class="t">        <span class="nam">query_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t322" href="#t322">322</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t323" href="#t323">323</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t324" href="#t324">324</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t325" href="#t325">325</a></span><span class="t">        <span class="nam">key_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t326" href="#t326">326</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t327" href="#t327">327</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t328" href="#t328">328</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t329" href="#t329">329</a></span><span class="t">        <span class="nam">value_input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t330" href="#t330">330</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t331" href="#t331">331</a></span><span class="t">            <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_model"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t332" href="#t332">332</a></span><span class="t">        <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t333" href="#t333">333</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t334" href="#t334">334</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t335" href="#t335">335</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t336" href="#t336">336</a></span><span class="t">        <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch kv_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t337" href="#t337">337</a></span><span class="t">    <span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t338" href="#t338">338</a></span><span class="t">        <span class="nam">attn_fn</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t339" href="#t339">339</a></span><span class="t">            <span class="nam">complex_attn_linear</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t340" href="#t340">340</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="key">or</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t341" href="#t341">341</a></span><span class="t">            <span class="key">else</span> <span class="nam">simple_attn_linear</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t342" href="#t342">342</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t343" href="#t343">343</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">343&#x202F;&#x219B;&#x202F;344</span><span class="annotate long">line 343 didn't jump to line 344, because the condition on line 343 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t344" href="#t344">344</a></span><span class="t">            <span class="nam">q</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_q</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t345" href="#t345">345</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t346" href="#t346">346</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t347" href="#t347">347</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t348" href="#t348">348</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t349" href="#t349">349</a></span><span class="t">                    <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t350" href="#t350">350</a></span><span class="t">                    <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">.</span><span class="nam">quant_state</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t351" href="#t351">351</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t352" href="#t352">352</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t353" href="#t353">353</a></span><span class="t">                    <span class="nam">query_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t354" href="#t354">354</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t355" href="#t355">355</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t356" href="#t356">356</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t357" href="#t357">357</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_Q</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t358" href="#t358">358</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t359" href="#t359">359</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t360" href="#t360">360</a></span><span class="t">            <span class="nam">q</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_q</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">query_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_Q</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t361" href="#t361">361</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">361&#x202F;&#x219B;&#x202F;362</span><span class="annotate long">line 361 didn't jump to line 362, because the condition on line 361 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t362" href="#t362">362</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">,</span> <span class="nam">Params4bit</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t363" href="#t363">363</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"W_K must be a Params4bit object if load_in_4bit is True"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t364" href="#t364">364</a></span><span class="t">            <span class="nam">k</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_k</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t365" href="#t365">365</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t366" href="#t366">366</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t367" href="#t367">367</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span> <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span> <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">quant_state</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t368" href="#t368">368</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t369" href="#t369">369</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t370" href="#t370">370</a></span><span class="t">                    <span class="nam">key_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t371" href="#t371">371</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t372" href="#t372">372</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t373" href="#t373">373</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t374" href="#t374">374</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_K</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t375" href="#t375">375</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t376" href="#t376">376</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t377" href="#t377">377</a></span><span class="t">            <span class="nam">k</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_k</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">key_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_K</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t378" href="#t378">378</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t379" href="#t379">379</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">379&#x202F;&#x219B;&#x202F;380</span><span class="annotate long">line 379 didn't jump to line 380, because the condition on line 379 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t380" href="#t380">380</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">Params4bit</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t381" href="#t381">381</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">"W_V must be a Params4bit object if load_in_4bit is True"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t382" href="#t382">382</a></span><span class="t">            <span class="nam">v</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_v</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t383" href="#t383">383</a></span><span class="t">                <span class="com"># call bitsandbytes method to dequantize and multiply</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t384" href="#t384">384</a></span><span class="t">                <span class="nam">bnb</span><span class="op">.</span><span class="nam">matmul_4bit</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t385" href="#t385">385</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t386" href="#t386">386</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">.</span><span class="nam">t</span><span class="op">(</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t387" href="#t387">387</a></span><span class="t">                    <span class="nam">bias</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t388" href="#t388">388</a></span><span class="t">                    <span class="nam">quant_state</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">.</span><span class="nam">quant_state</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t389" href="#t389">389</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">reshape</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t390" href="#t390">390</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t391" href="#t391">391</a></span><span class="t">                    <span class="nam">value_input</span><span class="op">.</span><span class="nam">shape</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t392" href="#t392">392</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t393" href="#t393">393</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_head</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t394" href="#t394">394</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t395" href="#t395">395</a></span><span class="t">                <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_V</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t396" href="#t396">396</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t397" href="#t397">397</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t398" href="#t398">398</a></span><span class="t">            <span class="nam">v</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_v</span><span class="op">(</span><span class="nam">attn_fn</span><span class="op">(</span><span class="nam">value_input</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b_V</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t399" href="#t399">399</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_attention_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">        <span class="nam">q</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="nam">k</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index query_pos key_pos"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">        <span class="nam">q_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">            <span class="nam">q</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head -> batch head_index query_pos d_head"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">        <span class="nam">k_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">            <span class="nam">k</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head -> batch head_index d_head key_pos"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">        <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">q_</span> <span class="op">@</span> <span class="nam">k_</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">attn_scale</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">412&#x202F;&#x219B;&#x202F;413</span><span class="annotate long">line 412 didn't jump to line 413, because the condition on line 412 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">            <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span> <span class="op">*</span> <span class="nam">F</span><span class="op">.</span><span class="nam">tanh</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">                <span class="nam">attn_scores</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">        <span class="key">return</span> <span class="nam">attn_scores</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_z_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">        <span class="nam">v</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="nam">pattern</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index query_pos key_pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="nam">v_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">            <span class="nam">v</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head -> batch head_index key_pos d_head"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">        <span class="nam">pattern_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">            <span class="nam">pattern</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">            <span class="str">"batch head_index query_pos key_pos -> batch head_index query_pos key_pos"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">        <span class="nam">z</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_z</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">            <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">                <span class="nam">pattern_</span> <span class="op">@</span> <span class="nam">v_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">                <span class="str">"batch head_index query_pos d_head -> batch query_pos head_index d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">        <span class="key">return</span> <span class="nam">z</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_causal_mask</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">        <span class="nam">attn_scores</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index pos pos_plus_past_kv_pos_offset"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">past_kv_pos_offset</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch offset_pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">        <span class="com"># The query context length is the number of positions we take queries from - if not using a past_kv_cache this is just the context length (for the current prompt), but if we're caching it can be different.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">        <span class="nam">query_ctx_length</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="com"># The key context length is the number of positions in the past - this includes all positions in the cache</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="com"># If not caching, query_ctx_length == key_ctx_length</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="nam">key_ctx_length</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">        <span class="key">if</span> <span class="nam">query_ctx_length</span> <span class="op">+</span> <span class="nam">past_kv_pos_offset</span> <span class="op">!=</span> <span class="nam">key_ctx_length</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">450&#x202F;&#x219B;&#x202F;451</span><span class="annotate long">line 450 didn't jump to line 451, because the condition on line 450 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">                <span class="str">f"query_ctx_length {query_ctx_length} + past_kv_pos_offset {past_kv_pos_offset} != key_ctx_length {key_ctx_length} - you likely have a bug."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">        <span class="com"># Index back to front to ensure local attention works</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">        <span class="nam">final_mask</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">mask</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">-</span><span class="nam">query_ctx_length</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="nam">key_ctx_length</span><span class="op">:</span><span class="op">]</span>  <span class="com"># [1, 1, pos, pos]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">            <span class="com"># Apply a causal mask to the attention scores considering the padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">            <span class="nam">einsum_str</span> <span class="op">=</span> <span class="str">"batch head pos offset_pos, batch offset_pos -> batch head pos offset_pos"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">            <span class="nam">final_mask</span> <span class="op">=</span> <span class="nam">final_mask</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">attention_mask</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">            <span class="nam">final_mask</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">einsum</span><span class="op">(</span><span class="nam">final_mask</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">,</span> <span class="nam">einsum_str</span><span class="op">)</span><span class="op">.</span><span class="nam">bool</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">        <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">final_mask</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">final_mask</span><span class="op">,</span> <span class="nam">attn_scores</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">IGNORE</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_sin_cos_rotary</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">        <span class="nam">rotary_dim</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">        <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">        <span class="nam">base</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">        <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx rotary_dim"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx rotary_dim"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t"><span class="str">        Calculate the sine and cosine waves to use in a rotary embedding. See https://blog.eleuther.ai/rotary-embeddings/ for details</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t"><span class="str">        Note: For some inexplicable reason, in GPT-J each ADJACENT pair of elements in k and q are rotated, in GPT-NeoX the pair of elements at k and k+n//2 are rotated (ie folding the full length in half, and then looking at pairs accordingly). I have absolutely no clue why, it should be completely equivalent.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t"><span class="str">        To resolve this, I've coded it to default to the GPT-J mode, but to explicitly check whether it's GPT-NeoX and then do the GPT-NeoX thing if it is.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">        <span class="nam">high_precision</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span> <span class="key">if</span> <span class="nam">dtype</span> <span class="op">!=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span> <span class="key">else</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">        <span class="nam">pos</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">high_precision</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">        <span class="nam">dim</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">rotary_dim</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">high_precision</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t">        <span class="com"># Llama-3.1 uses NTK-by-Parts Rotary Embedding introduced in Section 3.2 in https://arxiv.org/pdf/2309.00071</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">        <span class="com"># Implementation copied from https://github.com/huggingface/transformers/blob/v4.46.0/src/transformers/modeling_rope_utils.py#L310</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_NTK_by_parts_rope</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">485&#x202F;&#x219B;&#x202F;486</span><span class="annotate long">line 485 didn't jump to line 486, because the condition on line 485 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t">            <span class="nam">inv_freq</span> <span class="op">=</span> <span class="num">1.0</span> <span class="op">/</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t">                <span class="nam">base</span> <span class="op">**</span> <span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">rotary_dim</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">int64</span><span class="op">)</span><span class="op">.</span><span class="nam">float</span><span class="op">(</span><span class="op">)</span> <span class="op">/</span> <span class="nam">rotary_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">            <span class="nam">factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">            <span class="nam">low_freq_factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">            <span class="nam">high_freq_factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_high_freq_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">            <span class="nam">old_context_len</span> <span class="op">=</span> <span class="nam">n_ctx</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">            <span class="nam">low_freq_wavelen</span> <span class="op">=</span> <span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">            <span class="nam">high_freq_wavelen</span> <span class="op">=</span> <span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">high_freq_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">            <span class="nam">wavelen</span> <span class="op">=</span> <span class="num">2</span> <span class="op">*</span> <span class="nam">math</span><span class="op">.</span><span class="nam">pi</span> <span class="op">/</span> <span class="nam">inv_freq</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">            <span class="nam">inv_freq_llama</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">></span> <span class="nam">low_freq_wavelen</span><span class="op">,</span> <span class="nam">inv_freq</span> <span class="op">/</span> <span class="nam">factor</span><span class="op">,</span> <span class="nam">inv_freq</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">            <span class="nam">smooth_factor</span> <span class="op">=</span> <span class="op">(</span><span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">wavelen</span> <span class="op">-</span> <span class="nam">low_freq_factor</span><span class="op">)</span> <span class="op">/</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">                <span class="nam">high_freq_factor</span> <span class="op">-</span> <span class="nam">low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">            <span class="nam">smoothed_inv_freq</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">                <span class="num">1</span> <span class="op">-</span> <span class="nam">smooth_factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">            <span class="op">)</span> <span class="op">*</span> <span class="nam">inv_freq_llama</span> <span class="op">/</span> <span class="nam">factor</span> <span class="op">+</span> <span class="nam">smooth_factor</span> <span class="op">*</span> <span class="nam">inv_freq_llama</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">            <span class="nam">is_medium_freq</span> <span class="op">=</span> <span class="op">~</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">&lt;</span> <span class="nam">high_freq_wavelen</span><span class="op">)</span> <span class="op">*</span> <span class="op">~</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">></span> <span class="nam">low_freq_wavelen</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">            <span class="nam">inv_freq_llama</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">is_medium_freq</span><span class="op">,</span> <span class="nam">smoothed_inv_freq</span><span class="op">,</span> <span class="nam">inv_freq_llama</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">inv_freq_llama</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">base</span> <span class="op">**</span> <span class="op">(</span><span class="nam">dim</span> <span class="op">/</span> <span class="op">(</span><span class="nam">rotary_dim</span> <span class="op">/</span> <span class="num">2</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_adjacent_pairs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">510&#x202F;&#x219B;&#x202F;511</span><span class="annotate long">line 510 didn't jump to line 511, because the condition on line 510 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">repeat</span><span class="op">(</span><span class="nam">freq</span><span class="op">,</span> <span class="str">"d -> (d 2)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">repeat</span><span class="op">(</span><span class="nam">freq</span><span class="op">,</span> <span class="str">"d -> (2 d)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">        <span class="com"># Create a n_ctx x rotary_dim tensor, where each column is an arithmetic sequence of angles in that frequency</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">        <span class="nam">angles</span> <span class="op">=</span> <span class="nam">pos</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">/</span> <span class="nam">freq</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">sin</span><span class="op">(</span><span class="nam">angles</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cos</span><span class="op">(</span><span class="nam">angles</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">    <span class="key">def</span> <span class="nam">rotate_every_two</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"... rotary_dim"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"... rotary_dim"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t"><span class="str">        Rotary helper function, splits x into blocks of size 2 along the final axis and maps [x0, x1] to [-x1, x0]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t"><span class="str">        The final axis of x must have even length.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t"><span class="str">        GPT-NeoX and GPT-J do rotary subtly differently, see calculate_sin_cos_rotary for details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="nam">rot_x</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">clone</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_adjacent_pairs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">529&#x202F;&#x219B;&#x202F;530</span><span class="annotate long">line 529 didn't jump to line 530, because the condition on line 529 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span> <span class="op">=</span> <span class="op">-</span><span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t">            <span class="nam">n</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span> <span class="op">//</span> <span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="nam">n</span><span class="op">]</span> <span class="op">=</span> <span class="op">-</span><span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">n</span><span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">n</span><span class="op">:</span><span class="op">]</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="nam">n</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="key">return</span> <span class="nam">rot_x</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_rotary</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">        <span class="nam">x</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">        <span class="nam">past_kv_pos_offset</span><span class="op">=</span><span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch offset_pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">        <span class="com"># Only apply rotary to first rotary_dim dimensions (eg, if rotary_dim=64 and d_head=256, only apply to first 1/4 of dimensions)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">        <span class="nam">x_pos</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">        <span class="nam">x_rot</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">        <span class="nam">x_pass</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_dim</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="nam">x_flip</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotate_every_two</span><span class="op">(</span><span class="nam">x_rot</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">        <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">            <span class="nam">rotary_cos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">                <span class="key">None</span><span class="op">,</span> <span class="nam">past_kv_pos_offset</span> <span class="op">:</span> <span class="nam">past_kv_pos_offset</span> <span class="op">+</span> <span class="nam">x_pos</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">            <span class="nam">rotary_sin</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_sin</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">                <span class="key">None</span><span class="op">,</span> <span class="nam">past_kv_pos_offset</span> <span class="op">:</span> <span class="nam">past_kv_pos_offset</span> <span class="op">+</span> <span class="nam">x_pos</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">            <span class="nam">x_rotated</span> <span class="op">=</span> <span class="nam">x_rot</span> <span class="op">*</span> <span class="nam">rotary_cos</span> <span class="op">+</span> <span class="nam">x_flip</span> <span class="op">*</span> <span class="nam">rotary_sin</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">            <span class="nam">offset_position_ids</span> <span class="op">=</span> <span class="nam">get_offset_position_ids</span><span class="op">(</span><span class="nam">past_kv_pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="nam">offset_position_ids</span> <span class="op">=</span> <span class="nam">offset_position_ids</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">            <span class="nam">mask_rotary_cos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">[</span><span class="nam">offset_position_ids</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="nam">mask_rotary_sin</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_sin</span><span class="op">[</span><span class="nam">offset_position_ids</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="nam">x_rotated</span> <span class="op">=</span> <span class="nam">x_rot</span> <span class="op">*</span> <span class="nam">mask_rotary_cos</span> <span class="op">+</span> <span class="nam">x_flip</span> <span class="op">*</span> <span class="nam">mask_rotary_sin</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">x_rotated</span><span class="op">,</span> <span class="nam">x_pass</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_slope</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">        <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"query key"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">        <span class="str">"""Create an ALiBi Slope Matrix.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t"><span class="str">        Create the slope matrix used in ALiBi, before it is multiplied by the head-specific scalar.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t"><span class="str">        See :meth:`create_alibi_bias` for the full ALiBi bias calculation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_slope(3)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t"><span class="str">        tensor([[ 0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t"><span class="str">                [-1.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t"><span class="str">                [-2., -1.,  0.]])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t400" href="#t400">400</a></span><span class="t">        <span class="key">return</span> <span class="nam">q</span><span class="op">,</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t401" href="#t401">401</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t402" href="#t402">402</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_attention_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t403" href="#t403">403</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t404" href="#t404">404</a></span><span class="t">        <span class="nam">q</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t405" href="#t405">405</a></span><span class="t">        <span class="nam">k</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t406" href="#t406">406</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index query_pos key_pos"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t407" href="#t407">407</a></span><span class="t">        <span class="nam">q_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t408" href="#t408">408</a></span><span class="t">            <span class="nam">q</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head -> batch head_index query_pos d_head"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t409" href="#t409">409</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t410" href="#t410">410</a></span><span class="t">        <span class="nam">k_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t411" href="#t411">411</a></span><span class="t">            <span class="nam">k</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head -> batch head_index d_head key_pos"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t412" href="#t412">412</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t413" href="#t413">413</a></span><span class="t">        <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">q_</span> <span class="op">@</span> <span class="nam">k_</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">attn_scale</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t414" href="#t414">414</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">414&#x202F;&#x219B;&#x202F;415</span><span class="annotate long">line 414 didn't jump to line 415, because the condition on line 414 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t415" href="#t415">415</a></span><span class="t">            <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span> <span class="op">*</span> <span class="nam">F</span><span class="op">.</span><span class="nam">tanh</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t416" href="#t416">416</a></span><span class="t">                <span class="nam">attn_scores</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_scores_soft_cap</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t417" href="#t417">417</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t418" href="#t418">418</a></span><span class="t">        <span class="key">return</span> <span class="nam">attn_scores</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t419" href="#t419">419</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t420" href="#t420">420</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_z_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t421" href="#t421">421</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t422" href="#t422">422</a></span><span class="t">        <span class="nam">v</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t423" href="#t423">423</a></span><span class="t">        <span class="nam">pattern</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index query_pos key_pos"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t424" href="#t424">424</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch query_pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t425" href="#t425">425</a></span><span class="t">        <span class="nam">v_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t426" href="#t426">426</a></span><span class="t">            <span class="nam">v</span><span class="op">,</span> <span class="str">"batch key_pos head_index d_head -> batch head_index key_pos d_head"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t427" href="#t427">427</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t428" href="#t428">428</a></span><span class="t">        <span class="nam">pattern_</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t429" href="#t429">429</a></span><span class="t">            <span class="nam">pattern</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t430" href="#t430">430</a></span><span class="t">            <span class="str">"batch head_index query_pos key_pos -> batch head_index query_pos key_pos"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t431" href="#t431">431</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t432" href="#t432">432</a></span><span class="t">        <span class="nam">z</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">hook_z</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t433" href="#t433">433</a></span><span class="t">            <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t434" href="#t434">434</a></span><span class="t">                <span class="nam">pattern_</span> <span class="op">@</span> <span class="nam">v_</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t435" href="#t435">435</a></span><span class="t">                <span class="str">"batch head_index query_pos d_head -> batch query_pos head_index d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t436" href="#t436">436</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t437" href="#t437">437</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t438" href="#t438">438</a></span><span class="t">        <span class="key">return</span> <span class="nam">z</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t439" href="#t439">439</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t440" href="#t440">440</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_causal_mask</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t441" href="#t441">441</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t442" href="#t442">442</a></span><span class="t">        <span class="nam">attn_scores</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch head_index pos pos_plus_past_kv_pos_offset"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t443" href="#t443">443</a></span><span class="t">        <span class="nam">past_kv_pos_offset</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t444" href="#t444">444</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch offset_pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t445" href="#t445">445</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t446" href="#t446">446</a></span><span class="t">        <span class="com"># The query context length is the number of positions we take queries from - if not using a past_kv_cache this is just the context length (for the current prompt), but if we're caching it can be different.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t447" href="#t447">447</a></span><span class="t">        <span class="nam">query_ctx_length</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t448" href="#t448">448</a></span><span class="t">        <span class="com"># The key context length is the number of positions in the past - this includes all positions in the cache</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t449" href="#t449">449</a></span><span class="t">        <span class="com"># If not caching, query_ctx_length == key_ctx_length</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t450" href="#t450">450</a></span><span class="t">        <span class="nam">key_ctx_length</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t451" href="#t451">451</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t452" href="#t452">452</a></span><span class="t">        <span class="key">if</span> <span class="nam">query_ctx_length</span> <span class="op">+</span> <span class="nam">past_kv_pos_offset</span> <span class="op">!=</span> <span class="nam">key_ctx_length</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">452&#x202F;&#x219B;&#x202F;453</span><span class="annotate long">line 452 didn't jump to line 453, because the condition on line 452 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t453" href="#t453">453</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t454" href="#t454">454</a></span><span class="t">                <span class="str">f"query_ctx_length {query_ctx_length} + past_kv_pos_offset {past_kv_pos_offset} != key_ctx_length {key_ctx_length} - you likely have a bug."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t455" href="#t455">455</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t456" href="#t456">456</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t457" href="#t457">457</a></span><span class="t">        <span class="com"># Index back to front to ensure local attention works</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t458" href="#t458">458</a></span><span class="t">        <span class="nam">final_mask</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">mask</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">-</span><span class="nam">query_ctx_length</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="nam">key_ctx_length</span><span class="op">:</span><span class="op">]</span>  <span class="com"># [1, 1, pos, pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t459" href="#t459">459</a></span><span class="t">        <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t460" href="#t460">460</a></span><span class="t">            <span class="com"># Apply a causal mask to the attention scores considering the padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t461" href="#t461">461</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t462" href="#t462">462</a></span><span class="t">            <span class="com"># Add singleton dimensions to the attention mask to match the shape of the final mask</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t463" href="#t463">463</a></span><span class="t">            <span class="nam">attention_mask</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t464" href="#t464">464</a></span><span class="t">                <span class="nam">attention_mask</span><span class="op">,</span> <span class="str">"batch offset_pos -> batch 1 1 offset_pos"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t465" href="#t465">465</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t466" href="#t466">466</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t467" href="#t467">467</a></span><span class="t">            <span class="nam">final_mask</span> <span class="op">=</span> <span class="nam">final_mask</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">attention_mask</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t468" href="#t468">468</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t469" href="#t469">469</a></span><span class="t">            <span class="com"># Element-wise multiplication of the final mask and the attention mask and cast to boolean</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t470" href="#t470">470</a></span><span class="t">            <span class="nam">final_mask</span> <span class="op">=</span> <span class="op">(</span><span class="nam">final_mask</span> <span class="op">*</span> <span class="nam">attention_mask</span><span class="op">)</span><span class="op">.</span><span class="nam">bool</span><span class="op">(</span><span class="op">)</span>  <span class="com"># [batch, head, pos, offset_pos]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t471" href="#t471">471</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t472" href="#t472">472</a></span><span class="t">        <span class="nam">attn_scores</span> <span class="op">=</span> <span class="nam">attn_scores</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">final_mask</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t473" href="#t473">473</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">final_mask</span><span class="op">,</span> <span class="nam">attn_scores</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">IGNORE</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t474" href="#t474">474</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t475" href="#t475">475</a></span><span class="t">    <span class="key">def</span> <span class="nam">calculate_sin_cos_rotary</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t476" href="#t476">476</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t477" href="#t477">477</a></span><span class="t">        <span class="nam">rotary_dim</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t478" href="#t478">478</a></span><span class="t">        <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t479" href="#t479">479</a></span><span class="t">        <span class="nam">base</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10000</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t480" href="#t480">480</a></span><span class="t">        <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t481" href="#t481">481</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Tuple</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx rotary_dim"</span><span class="op">]</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx rotary_dim"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t482" href="#t482">482</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t483" href="#t483">483</a></span><span class="t"><span class="str">        Calculate the sine and cosine waves to use in a rotary embedding. See https://blog.eleuther.ai/rotary-embeddings/ for details</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t484" href="#t484">484</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t485" href="#t485">485</a></span><span class="t"><span class="str">        Note: For some inexplicable reason, in GPT-J each ADJACENT pair of elements in k and q are rotated, in GPT-NeoX the pair of elements at k and k+n//2 are rotated (ie folding the full length in half, and then looking at pairs accordingly). I have absolutely no clue why, it should be completely equivalent.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t486" href="#t486">486</a></span><span class="t"><span class="str">        To resolve this, I've coded it to default to the GPT-J mode, but to explicitly check whether it's GPT-NeoX and then do the GPT-NeoX thing if it is.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t487" href="#t487">487</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t488" href="#t488">488</a></span><span class="t">        <span class="nam">high_precision</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span> <span class="key">if</span> <span class="nam">dtype</span> <span class="op">!=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span> <span class="key">else</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t489" href="#t489">489</a></span><span class="t">        <span class="nam">pos</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">high_precision</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t490" href="#t490">490</a></span><span class="t">        <span class="nam">dim</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">rotary_dim</span> <span class="op">//</span> <span class="num">2</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">high_precision</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t491" href="#t491">491</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t492" href="#t492">492</a></span><span class="t">        <span class="com"># Llama-3.1 uses NTK-by-Parts Rotary Embedding introduced in Section 3.2 in https://arxiv.org/pdf/2309.00071</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t493" href="#t493">493</a></span><span class="t">        <span class="com"># Implementation copied from https://github.com/huggingface/transformers/blob/v4.46.0/src/transformers/modeling_rope_utils.py#L310</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t494" href="#t494">494</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_NTK_by_parts_rope</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">494&#x202F;&#x219B;&#x202F;495</span><span class="annotate long">line 494 didn't jump to line 495, because the condition on line 494 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t495" href="#t495">495</a></span><span class="t">            <span class="nam">inv_freq</span> <span class="op">=</span> <span class="num">1.0</span> <span class="op">/</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t496" href="#t496">496</a></span><span class="t">                <span class="nam">base</span> <span class="op">**</span> <span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">rotary_dim</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">int64</span><span class="op">)</span><span class="op">.</span><span class="nam">float</span><span class="op">(</span><span class="op">)</span> <span class="op">/</span> <span class="nam">rotary_dim</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t497" href="#t497">497</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t498" href="#t498">498</a></span><span class="t">            <span class="nam">factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t499" href="#t499">499</a></span><span class="t">            <span class="nam">low_freq_factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t500" href="#t500">500</a></span><span class="t">            <span class="nam">high_freq_factor</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">NTK_by_parts_high_freq_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t501" href="#t501">501</a></span><span class="t">            <span class="nam">old_context_len</span> <span class="op">=</span> <span class="nam">n_ctx</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t502" href="#t502">502</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t503" href="#t503">503</a></span><span class="t">            <span class="nam">low_freq_wavelen</span> <span class="op">=</span> <span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t504" href="#t504">504</a></span><span class="t">            <span class="nam">high_freq_wavelen</span> <span class="op">=</span> <span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">high_freq_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t505" href="#t505">505</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t506" href="#t506">506</a></span><span class="t">            <span class="nam">wavelen</span> <span class="op">=</span> <span class="num">2</span> <span class="op">*</span> <span class="nam">math</span><span class="op">.</span><span class="nam">pi</span> <span class="op">/</span> <span class="nam">inv_freq</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t507" href="#t507">507</a></span><span class="t">            <span class="nam">inv_freq_llama</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">></span> <span class="nam">low_freq_wavelen</span><span class="op">,</span> <span class="nam">inv_freq</span> <span class="op">/</span> <span class="nam">factor</span><span class="op">,</span> <span class="nam">inv_freq</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t508" href="#t508">508</a></span><span class="t">            <span class="nam">smooth_factor</span> <span class="op">=</span> <span class="op">(</span><span class="nam">old_context_len</span> <span class="op">/</span> <span class="nam">wavelen</span> <span class="op">-</span> <span class="nam">low_freq_factor</span><span class="op">)</span> <span class="op">/</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t509" href="#t509">509</a></span><span class="t">                <span class="nam">high_freq_factor</span> <span class="op">-</span> <span class="nam">low_freq_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t510" href="#t510">510</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t511" href="#t511">511</a></span><span class="t">            <span class="nam">smoothed_inv_freq</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t512" href="#t512">512</a></span><span class="t">                <span class="num">1</span> <span class="op">-</span> <span class="nam">smooth_factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t513" href="#t513">513</a></span><span class="t">            <span class="op">)</span> <span class="op">*</span> <span class="nam">inv_freq_llama</span> <span class="op">/</span> <span class="nam">factor</span> <span class="op">+</span> <span class="nam">smooth_factor</span> <span class="op">*</span> <span class="nam">inv_freq_llama</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t514" href="#t514">514</a></span><span class="t">            <span class="nam">is_medium_freq</span> <span class="op">=</span> <span class="op">~</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">&lt;</span> <span class="nam">high_freq_wavelen</span><span class="op">)</span> <span class="op">*</span> <span class="op">~</span><span class="op">(</span><span class="nam">wavelen</span> <span class="op">></span> <span class="nam">low_freq_wavelen</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t515" href="#t515">515</a></span><span class="t">            <span class="nam">inv_freq_llama</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">is_medium_freq</span><span class="op">,</span> <span class="nam">smoothed_inv_freq</span><span class="op">,</span> <span class="nam">inv_freq_llama</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t516" href="#t516">516</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">inv_freq_llama</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t517" href="#t517">517</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t518" href="#t518">518</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">base</span> <span class="op">**</span> <span class="op">(</span><span class="nam">dim</span> <span class="op">/</span> <span class="op">(</span><span class="nam">rotary_dim</span> <span class="op">/</span> <span class="num">2</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t519" href="#t519">519</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_adjacent_pairs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">519&#x202F;&#x219B;&#x202F;520</span><span class="annotate long">line 519 didn't jump to line 520, because the condition on line 519 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t520" href="#t520">520</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">repeat</span><span class="op">(</span><span class="nam">freq</span><span class="op">,</span> <span class="str">"d -> (d 2)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t521" href="#t521">521</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t522" href="#t522">522</a></span><span class="t">            <span class="nam">freq</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">repeat</span><span class="op">(</span><span class="nam">freq</span><span class="op">,</span> <span class="str">"d -> (2 d)"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t523" href="#t523">523</a></span><span class="t">        <span class="com"># Create a n_ctx x rotary_dim tensor, where each column is an arithmetic sequence of angles in that frequency</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t524" href="#t524">524</a></span><span class="t">        <span class="nam">angles</span> <span class="op">=</span> <span class="nam">pos</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">/</span> <span class="nam">freq</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t525" href="#t525">525</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">sin</span><span class="op">(</span><span class="nam">angles</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cos</span><span class="op">(</span><span class="nam">angles</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t526" href="#t526">526</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t527" href="#t527">527</a></span><span class="t">    <span class="key">def</span> <span class="nam">rotate_every_two</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t528" href="#t528">528</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"... rotary_dim"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t529" href="#t529">529</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"... rotary_dim"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t530" href="#t530">530</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t531" href="#t531">531</a></span><span class="t"><span class="str">        Rotary helper function, splits x into blocks of size 2 along the final axis and maps [x0, x1] to [-x1, x0]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t532" href="#t532">532</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t533" href="#t533">533</a></span><span class="t"><span class="str">        The final axis of x must have even length.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t534" href="#t534">534</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t535" href="#t535">535</a></span><span class="t"><span class="str">        GPT-NeoX and GPT-J do rotary subtly differently, see calculate_sin_cos_rotary for details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t536" href="#t536">536</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t537" href="#t537">537</a></span><span class="t">        <span class="nam">rot_x</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">clone</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t538" href="#t538">538</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_adjacent_pairs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">538&#x202F;&#x219B;&#x202F;539</span><span class="annotate long">line 538 didn't jump to line 539, because the condition on line 538 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t539" href="#t539">539</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span> <span class="op">=</span> <span class="op">-</span><span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t540" href="#t540">540</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="op">:</span><span class="num">2</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t541" href="#t541">541</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t542" href="#t542">542</a></span><span class="t">            <span class="nam">n</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span> <span class="op">//</span> <span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t543" href="#t543">543</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="nam">n</span><span class="op">]</span> <span class="op">=</span> <span class="op">-</span><span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">n</span><span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t544" href="#t544">544</a></span><span class="t">            <span class="nam">rot_x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">n</span><span class="op">:</span><span class="op">]</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span><span class="nam">n</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t545" href="#t545">545</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t546" href="#t546">546</a></span><span class="t">        <span class="key">return</span> <span class="nam">rot_x</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t547" href="#t547">547</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t548" href="#t548">548</a></span><span class="t">    <span class="key">def</span> <span class="nam">apply_rotary</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t549" href="#t549">549</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t550" href="#t550">550</a></span><span class="t">        <span class="nam">x</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t551" href="#t551">551</a></span><span class="t">        <span class="nam">past_kv_pos_offset</span><span class="op">=</span><span class="num">0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t552" href="#t552">552</a></span><span class="t">        <span class="nam">attention_mask</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch offset_pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t553" href="#t553">553</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos head_index d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t554" href="#t554">554</a></span><span class="t">        <span class="com"># Only apply rotary to first rotary_dim dimensions (eg, if rotary_dim=64 and d_head=256, only apply to first 1/4 of dimensions)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t555" href="#t555">555</a></span><span class="t">        <span class="nam">x_pos</span> <span class="op">=</span> <span class="nam">x</span><span class="op">.</span><span class="nam">size</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t556" href="#t556">556</a></span><span class="t">        <span class="nam">x_rot</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="op">:</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_dim</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t557" href="#t557">557</a></span><span class="t">        <span class="nam">x_pass</span> <span class="op">=</span> <span class="nam">x</span><span class="op">[</span><span class="op">...</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">rotary_dim</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t558" href="#t558">558</a></span><span class="t">        <span class="nam">x_flip</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotate_every_two</span><span class="op">(</span><span class="nam">x_rot</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t559" href="#t559">559</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t560" href="#t560">560</a></span><span class="t">        <span class="key">if</span> <span class="nam">attention_mask</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t561" href="#t561">561</a></span><span class="t">            <span class="nam">rotary_cos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t562" href="#t562">562</a></span><span class="t">                <span class="key">None</span><span class="op">,</span> <span class="nam">past_kv_pos_offset</span> <span class="op">:</span> <span class="nam">past_kv_pos_offset</span> <span class="op">+</span> <span class="nam">x_pos</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t563" href="#t563">563</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t564" href="#t564">564</a></span><span class="t">            <span class="nam">rotary_sin</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_sin</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t565" href="#t565">565</a></span><span class="t">                <span class="key">None</span><span class="op">,</span> <span class="nam">past_kv_pos_offset</span> <span class="op">:</span> <span class="nam">past_kv_pos_offset</span> <span class="op">+</span> <span class="nam">x_pos</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t566" href="#t566">566</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t567" href="#t567">567</a></span><span class="t">            <span class="nam">x_rotated</span> <span class="op">=</span> <span class="nam">x_rot</span> <span class="op">*</span> <span class="nam">rotary_cos</span> <span class="op">+</span> <span class="nam">x_flip</span> <span class="op">*</span> <span class="nam">rotary_sin</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t568" href="#t568">568</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t569" href="#t569">569</a></span><span class="t">            <span class="nam">offset_position_ids</span> <span class="op">=</span> <span class="nam">get_offset_position_ids</span><span class="op">(</span><span class="nam">past_kv_pos_offset</span><span class="op">,</span> <span class="nam">attention_mask</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t570" href="#t570">570</a></span><span class="t">            <span class="nam">offset_position_ids</span> <span class="op">=</span> <span class="nam">offset_position_ids</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t571" href="#t571">571</a></span><span class="t">            <span class="nam">mask_rotary_cos</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_cos</span><span class="op">[</span><span class="nam">offset_position_ids</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t572" href="#t572">572</a></span><span class="t">            <span class="nam">mask_rotary_sin</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">rotary_sin</span><span class="op">[</span><span class="nam">offset_position_ids</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t573" href="#t573">573</a></span><span class="t">            <span class="nam">x_rotated</span> <span class="op">=</span> <span class="nam">x_rot</span> <span class="op">*</span> <span class="nam">mask_rotary_cos</span> <span class="op">+</span> <span class="nam">x_flip</span> <span class="op">*</span> <span class="nam">mask_rotary_sin</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t574" href="#t574">574</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t575" href="#t575">575</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">x_rotated</span><span class="op">,</span> <span class="nam">x_pass</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t576" href="#t576">576</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t577" href="#t577">577</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t578" href="#t578">578</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_slope</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t579" href="#t579">579</a></span><span class="t">        <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t580" href="#t580">580</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"query key"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t581" href="#t581">581</a></span><span class="t">        <span class="str">"""Create an ALiBi Slope Matrix.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t582" href="#t582">582</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t583" href="#t583">583</a></span><span class="t"><span class="str">        Create the slope matrix used in ALiBi, before it is multiplied by the head-specific scalar.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t584" href="#t584">584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_slope(4)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t"><span class="str">        tensor([[ 0.,  0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">                [-1.,  0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t"><span class="str">                [-2., -1.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t"><span class="str">                [-3., -2., -1.,  0.]])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">            n_ctx: The maximum number of tokens in a prompt.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t585" href="#t585">585</a></span><span class="t"><span class="str">        See :meth:`create_alibi_bias` for the full ALiBi bias calculation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t586" href="#t586">586</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t587" href="#t587">587</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t588" href="#t588">588</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t589" href="#t589">589</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_slope(3)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t590" href="#t590">590</a></span><span class="t"><span class="str">        tensor([[ 0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t591" href="#t591">591</a></span><span class="t"><span class="str">                [-1.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t592" href="#t592">592</a></span><span class="t"><span class="str">                [-2., -1.,  0.]])</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t593" href="#t593">593</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t"><span class="str">            A tensor of shape (n_ctx, n_ctx), where the upper triangle is zero and the lower</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t"><span class="str">            triangle is decreasing by a constant slope of 1 (towards the bottom left corner).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t">        <span class="com"># set rows as [[0,1,2...]]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">        <span class="nam">rows</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t">        <span class="com"># Set cols as [[0],[1],[2]...]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">        <span class="nam">cols</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t">        <span class="com"># Use broadcasting to create the desired lower triangular part of the matrix</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t">        <span class="nam">slope_matrix</span> <span class="op">=</span> <span class="nam">rows</span> <span class="op">-</span> <span class="nam">cols</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">        <span class="com"># Use the clamp method to set all positive values (upper right triangle) to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">        <span class="key">return</span> <span class="nam">slope_matrix</span><span class="op">.</span><span class="nam">clamp</span><span class="op">(</span><span class="nam">max</span><span class="op">=</span><span class="num">0</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t594" href="#t594">594</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_slope(4)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t595" href="#t595">595</a></span><span class="t"><span class="str">        tensor([[ 0.,  0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t596" href="#t596">596</a></span><span class="t"><span class="str">                [-1.,  0.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t597" href="#t597">597</a></span><span class="t"><span class="str">                [-2., -1.,  0.,  0.],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t598" href="#t598">598</a></span><span class="t"><span class="str">                [-3., -2., -1.,  0.]])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t599" href="#t599">599</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t600" href="#t600">600</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t601" href="#t601">601</a></span><span class="t"><span class="str">            n_ctx: The maximum number of tokens in a prompt.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t602" href="#t602">602</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t603" href="#t603">603</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t604" href="#t604">604</a></span><span class="t"><span class="str">            A tensor of shape (n_ctx, n_ctx), where the upper triangle is zero and the lower</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t605" href="#t605">605</a></span><span class="t"><span class="str">            triangle is decreasing by a constant slope of 1 (towards the bottom left corner).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t606" href="#t606">606</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t607" href="#t607">607</a></span><span class="t">        <span class="com"># set rows as [[0,1,2...]]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t608" href="#t608">608</a></span><span class="t">        <span class="nam">rows</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t609" href="#t609">609</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_multipliers</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">        <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="str">"""Create the ALiBi Scalar Multipliers for each Head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t610" href="#t610">610</a></span><span class="t">        <span class="com"># Set cols as [[0],[1],[2]...]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t611" href="#t611">611</a></span><span class="t">        <span class="nam">cols</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t612" href="#t612">612</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t613" href="#t613">613</a></span><span class="t">        <span class="com"># Use broadcasting to create the desired lower triangular part of the matrix</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t614" href="#t614">614</a></span><span class="t">        <span class="nam">slope_matrix</span> <span class="op">=</span> <span class="nam">rows</span> <span class="op">-</span> <span class="nam">cols</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t615" href="#t615">615</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t"><span class="str">        For n heads, the set of multipliers (m) is the geometric sequence that starts at 2^(-8/n), and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t"><span class="str">        uses that same value as its ratio. For example, with 8 heads the values would be [1/(2^1),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t"><span class="str">        1/(2^2), ... , 1/(2^8)]. With 16 heads the values would be [1/(2^0.5), 1/(2^1), ... , 1/(2^8)].</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t"><span class="str">        See :meth:`create_alibi_bias` for the full ALiBi bias calculation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_multipliers(8)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t"><span class="str">        tensor([0.5000, 0.2500, 0.1250, 0.0625, 0.0312, 0.0156, 0.0078, 0.0039])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_multipliers(16)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t"><span class="str">        tensor([0.7071, 0.5000, 0.3536, 0.2500, 0.1768, 0.1250, 0.0884, 0.0625, 0.0442, 0.0312,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t"><span class="str">                0.0221, 0.0156, 0.0110, 0.0078, 0.0055, 0.0039])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t616" href="#t616">616</a></span><span class="t">        <span class="com"># Use the clamp method to set all positive values (upper right triangle) to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t617" href="#t617">617</a></span><span class="t">        <span class="key">return</span> <span class="nam">slope_matrix</span><span class="op">.</span><span class="nam">clamp</span><span class="op">(</span><span class="nam">max</span><span class="op">=</span><span class="num">0</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t618" href="#t618">618</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t619" href="#t619">619</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t620" href="#t620">620</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_multipliers</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t621" href="#t621">621</a></span><span class="t">        <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t622" href="#t622">622</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t623" href="#t623">623</a></span><span class="t">        <span class="str">"""Create the ALiBi Scalar Multipliers for each Head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t624" href="#t624">624</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t625" href="#t625">625</a></span><span class="t"><span class="str">        For n heads, the set of multipliers (m) is the geometric sequence that starts at 2^(-8/n), and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t626" href="#t626">626</a></span><span class="t"><span class="str">        uses that same value as its ratio. For example, with 8 heads the values would be [1/(2^1),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t627" href="#t627">627</a></span><span class="t"><span class="str">        1/(2^2), ... , 1/(2^8)]. With 16 heads the values would be [1/(2^0.5), 1/(2^1), ... , 1/(2^8)].</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t628" href="#t628">628</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t629" href="#t629">629</a></span><span class="t"><span class="str">        See :meth:`create_alibi_bias` for the full ALiBi bias calculation.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t630" href="#t630">630</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t"><span class="str">            n_heads: The number of heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t"><span class="str">            device: The device to create the tensor on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t"><span class="str">            A tensor of shape (n_heads,) containing the scalar multiplier for each head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t">        <span class="com"># Calculate the starting value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">        <span class="nam">start</span> <span class="op">=</span> <span class="num">2</span> <span class="op">**</span> <span class="op">(</span><span class="op">-</span><span class="num">8</span> <span class="op">/</span> <span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t">        <span class="com"># Generate the indices [0, 1, ..., n_heads-1]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t">        <span class="nam">indices</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_heads</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t631" href="#t631">631</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t632" href="#t632">632</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t633" href="#t633">633</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_multipliers(8)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t634" href="#t634">634</a></span><span class="t"><span class="str">        tensor([0.5000, 0.2500, 0.1250, 0.0625, 0.0312, 0.0156, 0.0078, 0.0039])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t635" href="#t635">635</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t636" href="#t636">636</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_multipliers(16)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t637" href="#t637">637</a></span><span class="t"><span class="str">        tensor([0.7071, 0.5000, 0.3536, 0.2500, 0.1768, 0.1250, 0.0884, 0.0625, 0.0442, 0.0312,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t638" href="#t638">638</a></span><span class="t"><span class="str">                0.0221, 0.0156, 0.0110, 0.0078, 0.0055, 0.0039])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t639" href="#t639">639</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t640" href="#t640">640</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t641" href="#t641">641</a></span><span class="t"><span class="str">            n_heads: The number of heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t642" href="#t642">642</a></span><span class="t"><span class="str">            device: The device to create the tensor on.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t643" href="#t643">643</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t">        <span class="com"># Compute the multipliers, with the starting value being the same as the ratio</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t">        <span class="nam">multipliers</span> <span class="op">=</span> <span class="nam">start</span> <span class="op">*</span> <span class="op">(</span><span class="nam">start</span><span class="op">**</span><span class="nam">indices</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">        <span class="key">return</span> <span class="nam">multipliers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">        <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx query key"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">        <span class="str">"""Create the ALiBi Bias for all Heads.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t"><span class="str">        Calculate the ALiBi bias (https://arxiv.org/pdf/2108.12409.pdf) for all heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t"><span class="str">        The broad idea behind ALiBi is to remove the positional encoding from the original transformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t"><span class="str">        model, and instead apply a bias to each attention score. This bias is proportional to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t"><span class="str">        distance between the query and key (i.e. it encourage paying less attention to more distant</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t"><span class="str">        tokens), and is added to the attention scores before the softmax. It is used in models such as</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t"><span class="str">        Bloom.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_bias(2, 4, torch.device('cpu'))</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t"><span class="str">        tensor([[[ 0.0000,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t"><span class="str">            [-0.0625,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t"><span class="str">            [-0.1250, -0.0625,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t"><span class="str">            [-0.1875, -0.1250, -0.0625,  0.0000]],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t"><span class="str">            [[ 0.0000,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t"><span class="str">            [-0.0039,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t"><span class="str">            [-0.0078, -0.0039,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t"><span class="str">            [-0.0117, -0.0078, -0.0039,  0.0000]]])</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t"><span class="str">            n_heads: The number of heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t"><span class="str">            n_ctx: The maximum number of tokens in a prompt.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="str">            device: The device to create the tensor on.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t"><span class="str">            The ALiBi bias that should be added to the attention scores before the softmax.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">        <span class="com"># Create the slope matrix</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t">        <span class="nam">slope</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"query key"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">AbstractAttention</span><span class="op">.</span><span class="nam">create_alibi_slope</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t">            <span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">        <span class="com"># Create the scalar multiplier for each head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t">        <span class="nam">multipliers</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">AbstractAttention</span><span class="op">.</span><span class="nam">create_alibi_multipliers</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t">            <span class="nam">n_heads</span><span class="op">,</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="com"># The ALiBi bias is then m * slope_matrix</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">        <span class="nam">alibi_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">einsum</span><span class="op">(</span><span class="str">"ij,k->kij"</span><span class="op">,</span> <span class="nam">slope</span><span class="op">,</span> <span class="nam">multipliers</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">        <span class="key">return</span> <span class="nam">alibi_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t644" href="#t644">644</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t645" href="#t645">645</a></span><span class="t"><span class="str">            A tensor of shape (n_heads,) containing the scalar multiplier for each head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t646" href="#t646">646</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t647" href="#t647">647</a></span><span class="t">        <span class="com"># Calculate the starting value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t648" href="#t648">648</a></span><span class="t">        <span class="nam">start</span> <span class="op">=</span> <span class="num">2</span> <span class="op">**</span> <span class="op">(</span><span class="op">-</span><span class="num">8</span> <span class="op">/</span> <span class="nam">n_heads</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t649" href="#t649">649</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t650" href="#t650">650</a></span><span class="t">        <span class="com"># Generate the indices [0, 1, ..., n_heads-1]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t651" href="#t651">651</a></span><span class="t">        <span class="nam">indices</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">n_heads</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t652" href="#t652">652</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t653" href="#t653">653</a></span><span class="t">        <span class="com"># Compute the multipliers, with the starting value being the same as the ratio</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t654" href="#t654">654</a></span><span class="t">        <span class="nam">multipliers</span> <span class="op">=</span> <span class="nam">start</span> <span class="op">*</span> <span class="op">(</span><span class="nam">start</span><span class="op">**</span><span class="nam">indices</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t655" href="#t655">655</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t656" href="#t656">656</a></span><span class="t">        <span class="key">return</span> <span class="nam">multipliers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t657" href="#t657">657</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t658" href="#t658">658</a></span><span class="t">    <span class="op">@</span><span class="nam">staticmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t659" href="#t659">659</a></span><span class="t">    <span class="key">def</span> <span class="nam">create_alibi_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t660" href="#t660">660</a></span><span class="t">        <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">device</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t661" href="#t661">661</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx query key"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t662" href="#t662">662</a></span><span class="t">        <span class="str">"""Create the ALiBi Bias for all Heads.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t663" href="#t663">663</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t664" href="#t664">664</a></span><span class="t"><span class="str">        Calculate the ALiBi bias (https://arxiv.org/pdf/2108.12409.pdf) for all heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t665" href="#t665">665</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t666" href="#t666">666</a></span><span class="t"><span class="str">        The broad idea behind ALiBi is to remove the positional encoding from the original transformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t667" href="#t667">667</a></span><span class="t"><span class="str">        model, and instead apply a bias to each attention score. This bias is proportional to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t668" href="#t668">668</a></span><span class="t"><span class="str">        distance between the query and key (i.e. it encourage paying less attention to more distant</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t669" href="#t669">669</a></span><span class="t"><span class="str">        tokens), and is added to the attention scores before the softmax. It is used in models such as</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t670" href="#t670">670</a></span><span class="t"><span class="str">        Bloom.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t671" href="#t671">671</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t672" href="#t672">672</a></span><span class="t"><span class="str">        Examples:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t673" href="#t673">673</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t674" href="#t674">674</a></span><span class="t"><span class="str">        >>> AbstractAttention.create_alibi_bias(2, 4, torch.device('cpu'))</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t675" href="#t675">675</a></span><span class="t"><span class="str">        tensor([[[ 0.0000,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t676" href="#t676">676</a></span><span class="t"><span class="str">            [-0.0625,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t677" href="#t677">677</a></span><span class="t"><span class="str">            [-0.1250, -0.0625,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t678" href="#t678">678</a></span><span class="t"><span class="str">            [-0.1875, -0.1250, -0.0625,  0.0000]],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t679" href="#t679">679</a></span><span class="t"><span class="str">            [[ 0.0000,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t680" href="#t680">680</a></span><span class="t"><span class="str">            [-0.0039,  0.0000,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t681" href="#t681">681</a></span><span class="t"><span class="str">            [-0.0078, -0.0039,  0.0000,  0.0000],</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t682" href="#t682">682</a></span><span class="t"><span class="str">            [-0.0117, -0.0078, -0.0039,  0.0000]]])</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t683" href="#t683">683</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t684" href="#t684">684</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t685" href="#t685">685</a></span><span class="t"><span class="str">            n_heads: The number of heads in a layer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t686" href="#t686">686</a></span><span class="t"><span class="str">            n_ctx: The maximum number of tokens in a prompt.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t687" href="#t687">687</a></span><span class="t"><span class="str">            device: The device to create the tensor on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t688" href="#t688">688</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t689" href="#t689">689</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t690" href="#t690">690</a></span><span class="t"><span class="str">            The ALiBi bias that should be added to the attention scores before the softmax.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t691" href="#t691">691</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t692" href="#t692">692</a></span><span class="t">        <span class="com"># Create the slope matrix</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t693" href="#t693">693</a></span><span class="t">        <span class="nam">slope</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"query key"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">AbstractAttention</span><span class="op">.</span><span class="nam">create_alibi_slope</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t694" href="#t694">694</a></span><span class="t">            <span class="nam">n_ctx</span><span class="op">,</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t695" href="#t695">695</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t696" href="#t696">696</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t697" href="#t697">697</a></span><span class="t">        <span class="com"># Create the scalar multiplier for each head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t698" href="#t698">698</a></span><span class="t">        <span class="nam">multipliers</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"head_idx"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">AbstractAttention</span><span class="op">.</span><span class="nam">create_alibi_multipliers</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t699" href="#t699">699</a></span><span class="t">            <span class="nam">n_heads</span><span class="op">,</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t700" href="#t700">700</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t701" href="#t701">701</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t702" href="#t702">702</a></span><span class="t">        <span class="com"># Add singleton dimensions to make shapes compatible for broadcasting:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t703" href="#t703">703</a></span><span class="t">        <span class="nam">slope</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">slope</span><span class="op">,</span> <span class="str">"query key -> 1 query key"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t704" href="#t704">704</a></span><span class="t">        <span class="nam">multipliers</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">multipliers</span><span class="op">,</span> <span class="str">"head_idx -> head_idx 1 1"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t705" href="#t705">705</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t706" href="#t706">706</a></span><span class="t">        <span class="com"># Element-wise multiplication of the slope and multipliers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t707" href="#t707">707</a></span><span class="t">        <span class="nam">alibi_bias</span> <span class="op">=</span> <span class="nam">multipliers</span> <span class="op">*</span> <span class="nam">slope</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t708" href="#t708">708</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t709" href="#t709">709</a></span><span class="t">        <span class="key">return</span> <span class="nam">alibi_bias</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -787,7 +800,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_attention_py.html b/_static/coverage/d_db46118ef83ad831_attention_py.html
index 1cfd7ce39..54cd8117a 100644
--- a/_static/coverage/d_db46118ef83ad831_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_block_py.html b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
index a8ac67d92..4061fbe61 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
index 4dc58acbf..6b0ff4772 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -143,7 +143,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
index 3fb1741ce..39853e365 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
@@ -55,8 +55,8 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">20 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">20<span class="text"> run</span></button>
+            <span class="text">22 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">22<span class="text"> run</span></button>
             <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">0<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
             <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">0<span class="text"> partial</span></button>
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -88,9 +88,9 @@ <h2>
     <p class="pln"><span class="n"><a id="t4" href="#t4">4</a></span><span class="t"><span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t"><span class="key">from</span> <span class="nam">typing</span> <span class="key">import</span> <span class="nam">Dict</span><span class="op">,</span> <span class="nam">Union</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span> <span class="key">as</span> <span class="nam">nn</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">from</span> <span class="nam">fancy_einsum</span> <span class="key">import</span> <span class="nam">einsum</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">einops</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">import</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">nn</span> <span class="key">as</span> <span class="nam">nn</span>&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t"><span class="key">from</span> <span class="nam">jaxtyping</span> <span class="key">import</span> <span class="nam">Float</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t"><span class="key">from</span> <span class="nam">transformer_lens</span><span class="op">.</span><span class="nam">components</span> <span class="key">import</span> <span class="nam">LayerNorm</span>&nbsp;</span><span class="r"></span></p>
@@ -111,17 +111,18 @@ <h2>
     <p class="run"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNorm</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="run"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">    <span class="key">def</span> <span class="nam">forward</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">resid</span><span class="op">:</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos d_model"</span><span class="op">]</span><span class="op">)</span> <span class="op">-></span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">            <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">                <span class="str">"batch pos d_model_in, d_model_out d_model_in -> batch pos d_model_out"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">                <span class="nam">resid</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">W</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">            <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">(</span><span class="nam">resid</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">ln</span><span class="op">(</span><span class="nam">resid</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t">        <span class="key">return</span> <span class="nam">resid</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t">        <span class="com"># Add singleton dimension for broadcasting</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">rearrange</span><span class="op">(</span><span class="nam">resid</span><span class="op">,</span> <span class="str">"batch pos d_model_in -> batch pos 1 d_model_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">        <span class="com"># Element-wise multiplication of W and resid</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">resid</span> <span class="op">*</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">        <span class="com"># Sum over d_model_in dimension and add bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">resid</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span> <span class="op">+</span> <span class="nam">self</span><span class="op">.</span><span class="nam">b</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">(</span><span class="nam">resid</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t">        <span class="nam">resid</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">ln</span><span class="op">(</span><span class="nam">resid</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t">        <span class="key">return</span> <span class="nam">resid</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -131,7 +132,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_embed_py.html b/_static/coverage/d_db46118ef83ad831_embed_py.html
index e992a7386..767eb2482 100644
--- a/_static/coverage/d_db46118ef83ad831_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -125,7 +125,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
index 7f582b77f..95dd9b30c 100644
--- a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -269,7 +269,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
index 76c9d05d9..9a5257d55 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -144,7 +144,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
index 7dd401199..d962c34ae 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
index 427f1416b..27f68afbb 100644
--- a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -160,7 +160,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
index 4cb5416fd..42aeccb4c 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
index a0234e3ed..0cbd2d219 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -136,7 +136,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
index 3b5e33c1f..6693d77bf 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -231,7 +231,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_block_py.html b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
index f8e6817b0..0f0d013f2 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -247,7 +247,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
index a82091d3f..f18fd7d5a 100644
--- a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
index 844535f21..44030eb62 100644
--- a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -300,7 +300,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_unembed_py.html b/_static/coverage/d_db46118ef83ad831_unembed_py.html
index b7c5bccd1..aef4ffa26 100644
--- a/_static/coverage/d_db46118ef83ad831_unembed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_unembed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -122,7 +122,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/index.html b/_static/coverage/index.html
index 4d6532499..ad85dec08 100644
--- a/_static/coverage/index.html
+++ b/_static/coverage/index.html
@@ -47,7 +47,7 @@ <h1>Coverage report:
         </form>
         <p class="text">
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
 </header>
@@ -67,12 +67,12 @@ <h1>Coverage report:
         <tbody>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_ActivationCache_py.html">transformer_lens/ActivationCache.py</a></td>
-                <td>288</td>
+                <td>289</td>
                 <td>7</td>
                 <td>0</td>
                 <td>168</td>
                 <td>10</td>
-                <td class="right" data-ratio="433 456">95%</td>
+                <td class="right" data-ratio="434 457">95%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_FactoredMatrix_py.html">transformer_lens/FactoredMatrix.py</a></td>
@@ -103,12 +103,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_HookedTransformer_py.html">transformer_lens/HookedTransformer.py</a></td>
-                <td>738</td>
+                <td>740</td>
                 <td>141</td>
                 <td>0</td>
                 <td>456</td>
                 <td>90</td>
-                <td class="right" data-ratio="895 1194">75%</td>
+                <td class="right" data-ratio="897 1196">75%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_HookedTransformerConfig_py.html">transformer_lens/HookedTransformerConfig.py</a></td>
@@ -121,12 +121,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_SVDInterpreter_py.html">transformer_lens/SVDInterpreter.py</a></td>
-                <td>55</td>
+                <td>54</td>
                 <td>1</td>
                 <td>0</td>
                 <td>16</td>
                 <td>1</td>
-                <td class="right" data-ratio="69 71">97%</td>
+                <td class="right" data-ratio="68 70">97%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14___init___py.html">transformer_lens/__init__.py</a></td>
@@ -148,12 +148,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_abstract_attention_py.html">transformer_lens/components/abstract_attention.py</a></td>
-                <td>236</td>
+                <td>240</td>
                 <td>41</td>
                 <td>0</td>
                 <td>86</td>
                 <td>19</td>
-                <td class="right" data-ratio="258 322">80%</td>
+                <td class="right" data-ratio="262 326">80%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_attention_py.html">transformer_lens/components/attention.py</a></td>
@@ -184,12 +184,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_bert_mlm_head_py.html">transformer_lens/components/bert_mlm_head.py</a></td>
-                <td>20</td>
+                <td>22</td>
                 <td>0</td>
                 <td>0</td>
                 <td>0</td>
                 <td>0</td>
-                <td class="right" data-ratio="20 20">100%</td>
+                <td class="right" data-ratio="22 22">100%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_db46118ef83ad831_embed_py.html">transformer_lens/components/embed.py</a></td>
@@ -395,8 +395,8 @@ <h1>Coverage report:
                 <td>112</td>
                 <td>0</td>
                 <td>216</td>
-                <td>52</td>
-                <td class="right" data-ratio="330 536">62%</td>
+                <td>51</td>
+                <td class="right" data-ratio="331 536">62%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_past_key_value_caching_py.html">transformer_lens/past_key_value_caching.py</a></td>
@@ -652,12 +652,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_b2114f845e0399b7_attention_py.html">transformer_lens/utilities/attention.py</a></td>
-                <td>10</td>
+                <td>14</td>
                 <td>0</td>
                 <td>0</td>
                 <td>0</td>
                 <td>0</td>
-                <td class="right" data-ratio="10 10">100%</td>
+                <td class="right" data-ratio="14 14">100%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_b2114f845e0399b7_devices_py.html">transformer_lens/utilities/devices.py</a></td>
@@ -681,12 +681,12 @@ <h1>Coverage report:
         <tfoot>
             <tr class="total">
                 <td class="name left">Total</td>
-                <td>5026</td>
+                <td>5038</td>
                 <td>1139</td>
                 <td>0</td>
                 <td>1986</td>
-                <td>337</td>
-                <td class="right" data-ratio="5194 7012">74%</td>
+                <td>336</td>
+                <td class="right" data-ratio="5207 7024">74%</td>
             </tr>
         </tfoot>
     </table>
@@ -698,7 +698,7 @@ <h1>Coverage report:
     <div class="content">
         <p>
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-11-19 14:42 +0000
+            created at 2024-12-14 00:54 +0000
         </p>
     </div>
     <aside class="hidden">
diff --git a/_static/coverage/status.json b/_static/coverage/status.json
index bfe03a988..976126417 100644
--- a/_static/coverage/status.json
+++ b/_static/coverage/status.json
@@ -1 +1 @@
-{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"9e688d29b5d7e756b245666804e5f24e","index":{"nums":[0,1,288,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"ecf3200c538017a5f4a2a6d8a4bf9c4f","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"c650b58ec9761da62868fda329891a0b","index":{"nums":[0,1,738,0,141,456,90,158],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"54ef75dda88a6bf77428fc230a984c66","index":{"nums":[0,1,135,0,8,40,7,7],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"8b9a2792be15afabc033f624683fed44","index":{"nums":[0,1,55,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"000785e246b8b83a6722683adc749e1b","index":{"nums":[0,1,236,0,41,86,19,23],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"0c5ab7d0820de68c654488b524f45c4d","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"52d623d476218710d87c4c7081b05665","index":{"nums":[0,1,55,0,0,20,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"521e7e01284a7150605bf1dae5d7d9f1","index":{"nums":[0,1,101,0,15,52,10,18],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"bef22159b240cbc374be3831b29b25ea","index":{"nums":[0,1,234,0,39,124,18,46],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"97d2e7a42e43215a820b8fafea92149c","index":{"nums":[0,1,320,0,112,216,52,94],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"4f2b989be42046944eb76e787f12f8a4","index":{"nums":[0,1,41,0,36,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"7f944a3a3a9cf8f8df0018d09fea2fae","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"d2b4d5cb90c365cbb3ef3981a81e5ae9","index":{"nums":[0,1,458,0,135,208,18,88],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
+{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"5ce55533b35bb3aea32c87d3444573f7","index":{"nums":[0,1,289,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"ecf3200c538017a5f4a2a6d8a4bf9c4f","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"894cd039c08fbb4b4023464453b62b01","index":{"nums":[0,1,740,0,141,456,90,158],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"54ef75dda88a6bf77428fc230a984c66","index":{"nums":[0,1,135,0,8,40,7,7],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"16a2ab71f04385f7b023d23a2bdef6d8","index":{"nums":[0,1,54,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"cfb43d36db80beb3b8f25c159abad3cc","index":{"nums":[0,1,240,0,41,86,19,23],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"13e34c2e7e8b18edf5fd230ccec94ead","index":{"nums":[0,1,22,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"52d623d476218710d87c4c7081b05665","index":{"nums":[0,1,55,0,0,20,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"521e7e01284a7150605bf1dae5d7d9f1","index":{"nums":[0,1,101,0,15,52,10,18],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"60d1e6604c991526f1d8a3906b8fd1c8","index":{"nums":[0,1,234,0,39,124,18,46],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"2873ff135731d2dd591a750cc83eb464","index":{"nums":[0,1,320,0,112,216,51,93],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"4f2b989be42046944eb76e787f12f8a4","index":{"nums":[0,1,41,0,36,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"a1d30d709e8539cc4515e76ad414a3ae","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"d2b4d5cb90c365cbb3ef3981a81e5ae9","index":{"nums":[0,1,458,0,135,208,18,88],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
diff --git a/generated/code/transformer_lens.hook_points.html b/generated/code/transformer_lens.hook_points.html
index 293aab7ae..e0ae27af8 100644
--- a/generated/code/transformer_lens.hook_points.html
+++ b/generated/code/transformer_lens.hook_points.html
@@ -537,7 +537,8 @@
 <li><p><strong>clear_contexts</strong> (<em>bool</em><em>, </em><em>optional</em>) – If True, clears hook contexts whenever hooks are reset.
 Defaults to False.</p></li>
 <li><p><strong>pos_slice</strong> – The slice to apply to the cache output. Defaults to None, do nothing.</p></li>
-<li><p><strong>**model_kwargs</strong> – Keyword arguments for the model.</p></li>
+<li><p><strong>**model_kwargs</strong> – Keyword arguments for the model’s forward function. See your related
+models forward pass for details as to what sort of arguments you can pass through.</p></li>
 </ul>
 </dd>
 <dt class="field-even">Returns<span class="colon">:</span></dt>
@@ -567,7 +568,8 @@
 <li><p><strong>clear_contexts</strong> (<em>bool</em>) – If True, clears hook contexts whenever hooks are reset. Default is
 False.</p></li>
 <li><p><strong>*model_args</strong> – Positional arguments for the model.</p></li>
-<li><p><strong>**model_kwargs</strong> – Keyword arguments for the model.</p></li>
+<li><p><strong>**model_kwargs</strong> – Keyword arguments for the model’s forward function. See your related
+models forward pass for details as to what sort of arguments you can pass through.</p></li>
 </ul>
 </dd>
 </dl>
diff --git a/generated/code/transformer_lens.loading_from_pretrained.html b/generated/code/transformer_lens.loading_from_pretrained.html
index 26491af61..3dd320b86 100644
--- a/generated/code/transformer_lens.loading_from_pretrained.html
+++ b/generated/code/transformer_lens.loading_from_pretrained.html
@@ -370,7 +370,7 @@
 
 <dl class="py data">
 <dt class="sig sig-object py" id="transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES">
-<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">OFFICIAL_MODEL_NAMES</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">['gpt2',</span> <span class="pre">'gpt2-medium',</span> <span class="pre">'gpt2-large',</span> <span class="pre">'gpt2-xl',</span> <span class="pre">'distilgpt2',</span> <span class="pre">'facebook/opt-125m',</span> <span class="pre">'facebook/opt-1.3b',</span> <span class="pre">'facebook/opt-2.7b',</span> <span class="pre">'facebook/opt-6.7b',</span> <span class="pre">'facebook/opt-13b',</span> <span class="pre">'facebook/opt-30b',</span> <span class="pre">'facebook/opt-66b',</span> <span class="pre">'EleutherAI/gpt-neo-125M',</span> <span class="pre">'EleutherAI/gpt-neo-1.3B',</span> <span class="pre">'EleutherAI/gpt-neo-2.7B',</span> <span class="pre">'EleutherAI/gpt-j-6B',</span> <span class="pre">'EleutherAI/gpt-neox-20b',</span> <span class="pre">'stanford-crfm/alias-gpt2-small-x21',</span> <span class="pre">'stanford-crfm/battlestar-gpt2-small-x49',</span> <span class="pre">'stanford-crfm/caprica-gpt2-small-x81',</span> <span class="pre">'stanford-crfm/darkmatter-gpt2-small-x343',</span> <span class="pre">'stanford-crfm/expanse-gpt2-small-x777',</span> <span class="pre">'stanford-crfm/arwen-gpt2-medium-x21',</span> <span class="pre">'stanford-crfm/beren-gpt2-medium-x49',</span> <span class="pre">'stanford-crfm/celebrimbor-gpt2-medium-x81',</span> <span class="pre">'stanford-crfm/durin-gpt2-medium-x343',</span> <span class="pre">'stanford-crfm/eowyn-gpt2-medium-x777',</span> <span class="pre">'EleutherAI/pythia-14m',</span> <span class="pre">'EleutherAI/pythia-31m',</span> <span class="pre">'EleutherAI/pythia-70m',</span> <span class="pre">'EleutherAI/pythia-160m',</span> <span class="pre">'EleutherAI/pythia-410m',</span> <span class="pre">'EleutherAI/pythia-1b',</span> <span class="pre">'EleutherAI/pythia-1.4b',</span> <span class="pre">'EleutherAI/pythia-2.8b',</span> <span class="pre">'EleutherAI/pythia-6.9b',</span> <span class="pre">'EleutherAI/pythia-12b',</span> <span class="pre">'EleutherAI/pythia-70m-deduped',</span> <span class="pre">'EleutherAI/pythia-160m-deduped',</span> <span class="pre">'EleutherAI/pythia-410m-deduped',</span> <span class="pre">'EleutherAI/pythia-1b-deduped',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped',</span> <span class="pre">'EleutherAI/pythia-12b-deduped',</span> <span class="pre">'EleutherAI/pythia-70m-v0',</span> <span class="pre">'EleutherAI/pythia-160m-v0',</span> <span class="pre">'EleutherAI/pythia-410m-v0',</span> <span class="pre">'EleutherAI/pythia-1b-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-v0',</span> <span class="pre">'EleutherAI/pythia-12b-v0',</span> <span class="pre">'EleutherAI/pythia-70m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-410m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-12b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-seed1',</span> <span class="pre">'EleutherAI/pythia-160m-seed2',</span> <span class="pre">'EleutherAI/pythia-160m-seed3',</span> <span class="pre">'NeelNanda/SoLU_1L_v9_old',</span> <span class="pre">'NeelNanda/SoLU_2L_v10_old',</span> <span class="pre">'NeelNanda/SoLU_4L_v11_old',</span> <span class="pre">'NeelNanda/SoLU_6L_v13_old',</span> <span class="pre">'NeelNanda/SoLU_8L_v21_old',</span> <span class="pre">'NeelNanda/SoLU_10L_v22_old',</span> <span class="pre">'NeelNanda/SoLU_12L_v23_old',</span> <span class="pre">'NeelNanda/SoLU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_6L768W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_8L1024W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_10L1280W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_12L1536W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr',</span> <span class="pre">'NeelNanda/SoLU_1L512W_Wiki_Finetune',</span> <span class="pre">'NeelNanda/SoLU_4L512W_Wiki_Finetune',</span> <span class="pre">'ArthurConmy/redwood_attn_2l',</span> <span class="pre">'llama-7b-hf',</span> <span class="pre">'llama-13b-hf',</span> <span class="pre">'llama-30b-hf',</span> <span class="pre">'llama-65b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-70b-chat-hf',</span> <span class="pre">'codellama/CodeLlama-7b-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Python-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Instruct-hf',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B-Instruct',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-1B',</span> <span class="pre">'meta-llama/Llama-3.2-3B',</span> <span class="pre">'meta-llama/Llama-3.2-1B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-3B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B',</span> <span class="pre">'meta-llama/Llama-3.1-8B',</span> <span class="pre">'meta-llama/Llama-3.1-8B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B-Instruct',</span> <span class="pre">'Baidicoot/Othello-GPT-Transformer-Lens',</span> <span class="pre">'bert-base-cased',</span> <span class="pre">'roneneldan/TinyStories-1M',</span> <span class="pre">'roneneldan/TinyStories-3M',</span> <span class="pre">'roneneldan/TinyStories-8M',</span> <span class="pre">'roneneldan/TinyStories-28M',</span> <span class="pre">'roneneldan/TinyStories-33M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-1M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-3M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-8M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-28M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-33M',</span> <span class="pre">'roneneldan/TinyStories-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-2Layers-33M',</span> <span class="pre">'roneneldan/TinyStories-Instuct-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-2Layers-33M',</span> <span class="pre">'stabilityai/stablelm-base-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-base-alpha-7b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-7b',</span> <span class="pre">'mistralai/Mistral-7B-v0.1',</span> <span class="pre">'mistralai/Mistral-7B-Instruct-v0.1',</span> <span class="pre">'mistralai/Mistral-Nemo-Base-2407',</span> <span class="pre">'mistralai/Mixtral-8x7B-v0.1',</span> <span class="pre">'mistralai/Mixtral-8x7B-Instruct-v0.1',</span> <span class="pre">'bigscience/bloom-560m',</span> <span class="pre">'bigscience/bloom-1b1',</span> <span class="pre">'bigscience/bloom-1b7',</span> <span class="pre">'bigscience/bloom-3b',</span> <span class="pre">'bigscience/bloom-7b1',</span> <span class="pre">'bigcode/santacoder',</span> <span class="pre">'Qwen/Qwen-1_8B',</span> <span class="pre">'Qwen/Qwen-7B',</span> <span class="pre">'Qwen/Qwen-14B',</span> <span class="pre">'Qwen/Qwen-1_8B-Chat',</span> <span class="pre">'Qwen/Qwen-7B-Chat',</span> <span class="pre">'Qwen/Qwen-14B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-0.5B',</span> <span class="pre">'Qwen/Qwen1.5-0.5B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-1.8B',</span> <span class="pre">'Qwen/Qwen1.5-1.8B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-4B',</span> <span class="pre">'Qwen/Qwen1.5-4B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-7B',</span> <span class="pre">'Qwen/Qwen1.5-7B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-14B',</span> <span class="pre">'Qwen/Qwen1.5-14B-Chat',</span> <span class="pre">'Qwen/Qwen2-0.5B',</span> <span class="pre">'Qwen/Qwen2-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-1.5B',</span> <span class="pre">'Qwen/Qwen2-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-7B',</span> <span class="pre">'Qwen/Qwen2-7B-Instruct',</span> <span class="pre">'microsoft/phi-1',</span> <span class="pre">'microsoft/phi-1_5',</span> <span class="pre">'microsoft/phi-2',</span> <span class="pre">'microsoft/Phi-3-mini-4k-instruct',</span> <span class="pre">'google/gemma-2b',</span> <span class="pre">'google/gemma-7b',</span> <span class="pre">'google/gemma-2b-it',</span> <span class="pre">'google/gemma-7b-it',</span> <span class="pre">'google/gemma-2-2b',</span> <span class="pre">'google/gemma-2-2b-it',</span> <span class="pre">'google/gemma-2-9b',</span> <span class="pre">'google/gemma-2-9b-it',</span> <span class="pre">'google/gemma-2-27b',</span> <span class="pre">'google/gemma-2-27b-it',</span> <span class="pre">'01-ai/Yi-6B',</span> <span class="pre">'01-ai/Yi-34B',</span> <span class="pre">'01-ai/Yi-6B-Chat',</span> <span class="pre">'01-ai/Yi-34B-Chat',</span> <span class="pre">'google-t5/t5-small',</span> <span class="pre">'google-t5/t5-base',</span> <span class="pre">'google-t5/t5-large',</span> <span class="pre">'ai-forever/mGPT']</span></em><a class="headerlink" href="#transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES" title="Permalink to this definition">#</a></dt>
+<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">OFFICIAL_MODEL_NAMES</span></span><em class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">['gpt2',</span> <span class="pre">'gpt2-medium',</span> <span class="pre">'gpt2-large',</span> <span class="pre">'gpt2-xl',</span> <span class="pre">'distilgpt2',</span> <span class="pre">'facebook/opt-125m',</span> <span class="pre">'facebook/opt-1.3b',</span> <span class="pre">'facebook/opt-2.7b',</span> <span class="pre">'facebook/opt-6.7b',</span> <span class="pre">'facebook/opt-13b',</span> <span class="pre">'facebook/opt-30b',</span> <span class="pre">'facebook/opt-66b',</span> <span class="pre">'EleutherAI/gpt-neo-125M',</span> <span class="pre">'EleutherAI/gpt-neo-1.3B',</span> <span class="pre">'EleutherAI/gpt-neo-2.7B',</span> <span class="pre">'EleutherAI/gpt-j-6B',</span> <span class="pre">'EleutherAI/gpt-neox-20b',</span> <span class="pre">'stanford-crfm/alias-gpt2-small-x21',</span> <span class="pre">'stanford-crfm/battlestar-gpt2-small-x49',</span> <span class="pre">'stanford-crfm/caprica-gpt2-small-x81',</span> <span class="pre">'stanford-crfm/darkmatter-gpt2-small-x343',</span> <span class="pre">'stanford-crfm/expanse-gpt2-small-x777',</span> <span class="pre">'stanford-crfm/arwen-gpt2-medium-x21',</span> <span class="pre">'stanford-crfm/beren-gpt2-medium-x49',</span> <span class="pre">'stanford-crfm/celebrimbor-gpt2-medium-x81',</span> <span class="pre">'stanford-crfm/durin-gpt2-medium-x343',</span> <span class="pre">'stanford-crfm/eowyn-gpt2-medium-x777',</span> <span class="pre">'EleutherAI/pythia-14m',</span> <span class="pre">'EleutherAI/pythia-31m',</span> <span class="pre">'EleutherAI/pythia-70m',</span> <span class="pre">'EleutherAI/pythia-160m',</span> <span class="pre">'EleutherAI/pythia-410m',</span> <span class="pre">'EleutherAI/pythia-1b',</span> <span class="pre">'EleutherAI/pythia-1.4b',</span> <span class="pre">'EleutherAI/pythia-2.8b',</span> <span class="pre">'EleutherAI/pythia-6.9b',</span> <span class="pre">'EleutherAI/pythia-12b',</span> <span class="pre">'EleutherAI/pythia-70m-deduped',</span> <span class="pre">'EleutherAI/pythia-160m-deduped',</span> <span class="pre">'EleutherAI/pythia-410m-deduped',</span> <span class="pre">'EleutherAI/pythia-1b-deduped',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped',</span> <span class="pre">'EleutherAI/pythia-12b-deduped',</span> <span class="pre">'EleutherAI/pythia-70m-v0',</span> <span class="pre">'EleutherAI/pythia-160m-v0',</span> <span class="pre">'EleutherAI/pythia-410m-v0',</span> <span class="pre">'EleutherAI/pythia-1b-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-v0',</span> <span class="pre">'EleutherAI/pythia-12b-v0',</span> <span class="pre">'EleutherAI/pythia-70m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-410m-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-1.4b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-2.8b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-6.9b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-12b-deduped-v0',</span> <span class="pre">'EleutherAI/pythia-160m-seed1',</span> <span class="pre">'EleutherAI/pythia-160m-seed2',</span> <span class="pre">'EleutherAI/pythia-160m-seed3',</span> <span class="pre">'NeelNanda/SoLU_1L_v9_old',</span> <span class="pre">'NeelNanda/SoLU_2L_v10_old',</span> <span class="pre">'NeelNanda/SoLU_4L_v11_old',</span> <span class="pre">'NeelNanda/SoLU_6L_v13_old',</span> <span class="pre">'NeelNanda/SoLU_8L_v21_old',</span> <span class="pre">'NeelNanda/SoLU_10L_v22_old',</span> <span class="pre">'NeelNanda/SoLU_12L_v23_old',</span> <span class="pre">'NeelNanda/SoLU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_6L768W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_8L1024W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_10L1280W_C4_Code',</span> <span class="pre">'NeelNanda/SoLU_12L1536W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/GELU_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_1L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_2L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_3L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn_Only_4L512W_C4_Code',</span> <span class="pre">'NeelNanda/Attn-Only-2L512W-Shortformer-6B-big-lr',</span> <span class="pre">'NeelNanda/SoLU_1L512W_Wiki_Finetune',</span> <span class="pre">'NeelNanda/SoLU_4L512W_Wiki_Finetune',</span> <span class="pre">'ArthurConmy/redwood_attn_2l',</span> <span class="pre">'llama-7b-hf',</span> <span class="pre">'llama-13b-hf',</span> <span class="pre">'llama-30b-hf',</span> <span class="pre">'llama-65b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-hf',</span> <span class="pre">'meta-llama/Llama-2-7b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-hf',</span> <span class="pre">'meta-llama/Llama-2-13b-chat-hf',</span> <span class="pre">'meta-llama/Llama-2-70b-chat-hf',</span> <span class="pre">'codellama/CodeLlama-7b-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Python-hf',</span> <span class="pre">'codellama/CodeLlama-7b-Instruct-hf',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B',</span> <span class="pre">'meta-llama/Meta-Llama-3-8B-Instruct',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B',</span> <span class="pre">'meta-llama/Meta-Llama-3-70B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-1B',</span> <span class="pre">'meta-llama/Llama-3.2-3B',</span> <span class="pre">'meta-llama/Llama-3.2-1B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.2-3B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B',</span> <span class="pre">'meta-llama/Llama-3.1-8B',</span> <span class="pre">'meta-llama/Llama-3.1-8B-Instruct',</span> <span class="pre">'meta-llama/Llama-3.1-70B-Instruct',</span> <span class="pre">'Baidicoot/Othello-GPT-Transformer-Lens',</span> <span class="pre">'bert-base-cased',</span> <span class="pre">'roneneldan/TinyStories-1M',</span> <span class="pre">'roneneldan/TinyStories-3M',</span> <span class="pre">'roneneldan/TinyStories-8M',</span> <span class="pre">'roneneldan/TinyStories-28M',</span> <span class="pre">'roneneldan/TinyStories-33M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-1M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-3M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-8M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-28M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-33M',</span> <span class="pre">'roneneldan/TinyStories-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-2Layers-33M',</span> <span class="pre">'roneneldan/TinyStories-Instuct-1Layer-21M',</span> <span class="pre">'roneneldan/TinyStories-Instruct-2Layers-33M',</span> <span class="pre">'stabilityai/stablelm-base-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-base-alpha-7b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-3b',</span> <span class="pre">'stabilityai/stablelm-tuned-alpha-7b',</span> <span class="pre">'mistralai/Mistral-7B-v0.1',</span> <span class="pre">'mistralai/Mistral-7B-Instruct-v0.1',</span> <span class="pre">'mistralai/Mistral-Nemo-Base-2407',</span> <span class="pre">'mistralai/Mixtral-8x7B-v0.1',</span> <span class="pre">'mistralai/Mixtral-8x7B-Instruct-v0.1',</span> <span class="pre">'bigscience/bloom-560m',</span> <span class="pre">'bigscience/bloom-1b1',</span> <span class="pre">'bigscience/bloom-1b7',</span> <span class="pre">'bigscience/bloom-3b',</span> <span class="pre">'bigscience/bloom-7b1',</span> <span class="pre">'bigcode/santacoder',</span> <span class="pre">'Qwen/Qwen-1_8B',</span> <span class="pre">'Qwen/Qwen-7B',</span> <span class="pre">'Qwen/Qwen-14B',</span> <span class="pre">'Qwen/Qwen-1_8B-Chat',</span> <span class="pre">'Qwen/Qwen-7B-Chat',</span> <span class="pre">'Qwen/Qwen-14B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-0.5B',</span> <span class="pre">'Qwen/Qwen1.5-0.5B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-1.8B',</span> <span class="pre">'Qwen/Qwen1.5-1.8B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-4B',</span> <span class="pre">'Qwen/Qwen1.5-4B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-7B',</span> <span class="pre">'Qwen/Qwen1.5-7B-Chat',</span> <span class="pre">'Qwen/Qwen1.5-14B',</span> <span class="pre">'Qwen/Qwen1.5-14B-Chat',</span> <span class="pre">'Qwen/Qwen2-0.5B',</span> <span class="pre">'Qwen/Qwen2-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-1.5B',</span> <span class="pre">'Qwen/Qwen2-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2-7B',</span> <span class="pre">'Qwen/Qwen2-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-0.5B',</span> <span class="pre">'Qwen/Qwen2.5-0.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-1.5B',</span> <span class="pre">'Qwen/Qwen2.5-1.5B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-3B',</span> <span class="pre">'Qwen/Qwen2.5-3B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-7B',</span> <span class="pre">'Qwen/Qwen2.5-7B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-14B',</span> <span class="pre">'Qwen/Qwen2.5-14B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-32B',</span> <span class="pre">'Qwen/Qwen2.5-32B-Instruct',</span> <span class="pre">'Qwen/Qwen2.5-72B',</span> <span class="pre">'Qwen/Qwen2.5-72B-Instruct',</span> <span class="pre">'Qwen/QwQ-32B-Preview',</span> <span class="pre">'microsoft/phi-1',</span> <span class="pre">'microsoft/phi-1_5',</span> <span class="pre">'microsoft/phi-2',</span> <span class="pre">'microsoft/Phi-3-mini-4k-instruct',</span> <span class="pre">'google/gemma-2b',</span> <span class="pre">'google/gemma-7b',</span> <span class="pre">'google/gemma-2b-it',</span> <span class="pre">'google/gemma-7b-it',</span> <span class="pre">'google/gemma-2-2b',</span> <span class="pre">'google/gemma-2-2b-it',</span> <span class="pre">'google/gemma-2-9b',</span> <span class="pre">'google/gemma-2-9b-it',</span> <span class="pre">'google/gemma-2-27b',</span> <span class="pre">'google/gemma-2-27b-it',</span> <span class="pre">'01-ai/Yi-6B',</span> <span class="pre">'01-ai/Yi-34B',</span> <span class="pre">'01-ai/Yi-6B-Chat',</span> <span class="pre">'01-ai/Yi-34B-Chat',</span> <span class="pre">'google-t5/t5-small',</span> <span class="pre">'google-t5/t5-base',</span> <span class="pre">'google-t5/t5-large',</span> <span class="pre">'ai-forever/mGPT']</span></em><a class="headerlink" href="#transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES" title="Permalink to this definition">#</a></dt>
 <dd><p>Official model names for models on HuggingFace.</p>
 </dd></dl>
 
diff --git a/generated/demos/Exploratory_Analysis_Demo.html b/generated/demos/Exploratory_Analysis_Demo.html
index c4acc3aab..596aa96c4 100644
--- a/generated/demos/Exploratory_Analysis_Demo.html
+++ b/generated/demos/Exploratory_Analysis_Demo.html
@@ -476,37 +476,37 @@ <h3>Indirect Object Identification<a class="headerlink" href="#Indirect-Object-I
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "adf78455ad4b4255b5ee12b111c8d6bf"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2566b58370ab4ee1a23ec40b298e524d"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f2a3e2ebfaad4d4d8b38096dac3f0a14"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "4c4a1ec2b7514c1297506faae67c1374"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ea70744b24bb4f8797907ee24a3e69c9"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "54e358754fd241a7beb89dbd18d54469"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "d682c8c5f56443c38680f259b4f0faef"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "c9ac7e38eb534678896df289998d1855"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "3790ba28df6f45f098a68fcec3f968f8"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "0b727299d6b94848ba14d79205e0d847"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "1fd27b66709b400bae1bba1dd1353b37"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "cf0868984e46408fb4a1d566ec3db0b0"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -899,9 +899,9 @@ <h3>Logit Lens<a class="headerlink" href="#Logit-Lens" title="Permalink to this
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="efc11a98-bb03-49b5-8a98-2d19f7fe591b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("efc11a98-bb03-49b5-8a98-2d19f7fe591b")) {                    Plotly.newPlot(                        "efc11a98-bb03-49b5-8a98-2d19f7fe591b",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="49047634-c443-4608-b07a-015a3c6cf28b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("49047634-c443-4608-b07a-015a3c6cf28b")) {                    Plotly.newPlot(                        "49047634-c443-4608-b07a-015a3c6cf28b",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('efc11a98-bb03-49b5-8a98-2d19f7fe591b');
+var gd = document.getElementById('49047634-c443-4608-b07a-015a3c6cf28b');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -947,9 +947,9 @@ <h3>Layer Attribution<a class="headerlink" href="#Layer-Attribution" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="9cfc6e76-339b-42f8-a6db-94e90576fc5e" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("9cfc6e76-339b-42f8-a6db-94e90576fc5e")) {                    Plotly.newPlot(                        "9cfc6e76-339b-42f8-a6db-94e90576fc5e",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="619fc2fb-6643-474d-9104-a4fc84996d47" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("619fc2fb-6643-474d-9104-a4fc84996d47")) {                    Plotly.newPlot(                        "619fc2fb-6643-474d-9104-a4fc84996d47",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('9cfc6e76-339b-42f8-a6db-94e90576fc5e');
+var gd = document.getElementById('619fc2fb-6643-474d-9104-a4fc84996d47');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1017,9 +1017,9 @@ <h2>Head Attribution<a class="headerlink" href="#Head-Attribution" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d6688b13-6cd5-49b7-97f4-6d2139a939a2" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d6688b13-6cd5-49b7-97f4-6d2139a939a2")) {                    Plotly.newPlot(                        "d6688b13-6cd5-49b7-97f4-6d2139a939a2",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="d13d12cd-e58c-4918-adf2-4cba0fcd4cfb" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d13d12cd-e58c-4918-adf2-4cba0fcd4cfb")) {                    Plotly.newPlot(                        "d13d12cd-e58c-4918-adf2-4cba0fcd4cfb",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d6688b13-6cd5-49b7-97f4-6d2139a939a2');
+var gd = document.getElementById('d13d12cd-e58c-4918-adf2-4cba0fcd4cfb');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1141,19 +1141,19 @@ <h2>Attention Analysis<a class="headerlink" href="#Attention-Analysis" title="Pe
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-8c2f229e-31c6" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-238275f7-4d84" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-8c2f229e-31c6",
+      "circuits-vis-238275f7-4d84",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], "attentionHeadNames": ["L9H9", "L9H6", "L10H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-d75cc7e1-d317" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-a59ac87b-fe56" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-d75cc7e1-d317",
+      "circuits-vis-a59ac87b-fe56",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], "attentionHeadNames": ["L10H7", "L11H10", "L11H2"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1306,9 +1306,9 @@ <h2>Residual Stream<a class="headerlink" href="#Residual-Stream" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d82fdf01-8df1-4af3-b10a-eb8e5065afdc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d82fdf01-8df1-4af3-b10a-eb8e5065afdc")) {                    Plotly.newPlot(                        "d82fdf01-8df1-4af3-b10a-eb8e5065afdc",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="4bc9653c-0aeb-4930-9e54-0fc92a4e0591" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4bc9653c-0aeb-4930-9e54-0fc92a4e0591")) {                    Plotly.newPlot(                        "4bc9653c-0aeb-4930-9e54-0fc92a4e0591",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d82fdf01-8df1-4af3-b10a-eb8e5065afdc');
+var gd = document.getElementById('4bc9653c-0aeb-4930-9e54-0fc92a4e0591');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1394,9 +1394,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="e0f8d107-6766-415c-8f1a-09e327e99d13" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e0f8d107-6766-415c-8f1a-09e327e99d13")) {                    Plotly.newPlot(                        "e0f8d107-6766-415c-8f1a-09e327e99d13",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="191ca322-bcdc-49ad-a055-e5a33b6bafa3" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("191ca322-bcdc-49ad-a055-e5a33b6bafa3")) {                    Plotly.newPlot(                        "191ca322-bcdc-49ad-a055-e5a33b6bafa3",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e0f8d107-6766-415c-8f1a-09e327e99d13');
+var gd = document.getElementById('191ca322-bcdc-49ad-a055-e5a33b6bafa3');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1443,9 +1443,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="611a846b-5fc0-4193-8bf8-9009ceffb348" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("611a846b-5fc0-4193-8bf8-9009ceffb348")) {                    Plotly.newPlot(                        "611a846b-5fc0-4193-8bf8-9009ceffb348",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="53f8687c-322d-426d-8a84-d3e680bc0251" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("53f8687c-322d-426d-8a84-d3e680bc0251")) {                    Plotly.newPlot(                        "53f8687c-322d-426d-8a84-d3e680bc0251",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('611a846b-5fc0-4193-8bf8-9009ceffb348');
+var gd = document.getElementById('53f8687c-322d-426d-8a84-d3e680bc0251');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1526,9 +1526,9 @@ <h2>Heads<a class="headerlink" href="#Heads" title="Permalink to this heading">#
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="f5742c28-dd8a-48dc-a7e5-91b761e718a8" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f5742c28-dd8a-48dc-a7e5-91b761e718a8")) {                    Plotly.newPlot(                        "f5742c28-dd8a-48dc-a7e5-91b761e718a8",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="d7db9d56-c782-4460-85c8-5165d9b90be4" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d7db9d56-c782-4460-85c8-5165d9b90be4")) {                    Plotly.newPlot(                        "d7db9d56-c782-4460-85c8-5165d9b90be4",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f5742c28-dd8a-48dc-a7e5-91b761e718a8');
+var gd = document.getElementById('d7db9d56-c782-4460-85c8-5165d9b90be4');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1599,9 +1599,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="41ce7e50-d3c8-4c1e-9792-e60a432c716e" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("41ce7e50-d3c8-4c1e-9792-e60a432c716e")) {                    Plotly.newPlot(                        "41ce7e50-d3c8-4c1e-9792-e60a432c716e",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="b78bb017-f4f8-4418-b905-eac4d9764ea8" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b78bb017-f4f8-4418-b905-eac4d9764ea8")) {                    Plotly.newPlot(                        "b78bb017-f4f8-4418-b905-eac4d9764ea8",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('41ce7e50-d3c8-4c1e-9792-e60a432c716e');
+var gd = document.getElementById('b78bb017-f4f8-4418-b905-eac4d9764ea8');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1655,9 +1655,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="524094c7-fa65-4920-a689-538877ae3454" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("524094c7-fa65-4920-a689-538877ae3454")) {                    Plotly.newPlot(                        "524094c7-fa65-4920-a689-538877ae3454",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="c2250b6e-3006-4a68-a458-2890b3c0d4df" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c2250b6e-3006-4a68-a458-2890b3c0d4df")) {                    Plotly.newPlot(                        "c2250b6e-3006-4a68-a458-2890b3c0d4df",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('524094c7-fa65-4920-a689-538877ae3454');
+var gd = document.getElementById('c2250b6e-3006-4a68-a458-2890b3c0d4df');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1744,9 +1744,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="06bfd2af-c00f-4a8b-93e6-4e78e590b1b5" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("06bfd2af-c00f-4a8b-93e6-4e78e590b1b5")) {                    Plotly.newPlot(                        "06bfd2af-c00f-4a8b-93e6-4e78e590b1b5",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="f21b691b-6def-4049-8b21-6c417fc5a93c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f21b691b-6def-4049-8b21-6c417fc5a93c")) {                    Plotly.newPlot(                        "f21b691b-6def-4049-8b21-6c417fc5a93c",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('06bfd2af-c00f-4a8b-93e6-4e78e590b1b5');
+var gd = document.getElementById('f21b691b-6def-4049-8b21-6c417fc5a93c');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1774,9 +1774,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="c933b3d0-ef0f-4d9d-8702-95ead096afef" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c933b3d0-ef0f-4d9d-8702-95ead096afef")) {                    Plotly.newPlot(                        "c933b3d0-ef0f-4d9d-8702-95ead096afef",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="d6f23061-00f8-4a84-9d17-cede03fea051" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d6f23061-00f8-4a84-9d17-cede03fea051")) {                    Plotly.newPlot(                        "d6f23061-00f8-4a84-9d17-cede03fea051",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('c933b3d0-ef0f-4d9d-8702-95ead096afef');
+var gd = document.getElementById('d6f23061-00f8-4a84-9d17-cede03fea051');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1853,27 +1853,27 @@ <h3>Visualizing Attention Patterns<a class="headerlink" href="#Visualizing-Atten
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-1a7d2574-bd46" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-cec38c2d-008b" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-1a7d2574-bd46",
+      "circuits-vis-cec38c2d-008b",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], "attentionHeadNames": ["L5H5", "L6H9", "L3H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-54ca365f-dfb7" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-4796b044-6a22" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-54ca365f-dfb7",
+      "circuits-vis-4796b044-6a22",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], "attentionHeadNames": ["L8H6", "L8H10", "L7H9", "L7H3"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-40e7b9a3-aa19" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-3f1d1fde-8de5" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-40e7b9a3-aa19",
+      "circuits-vis-3f1d1fde-8de5",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], "attentionHeadNames": ["L10H7", "L11H10", "L9H9"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1955,11 +1955,11 @@ <h3>Early Heads are Induction Heads(?!)<a class="headerlink" href="#Early-Heads-
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-c41df250-c3be" style="margin: 15px 0;"/>
+<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-2ebcde31-65b5" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-c41df250-c3be",
+      "circuits-vis-2ebcde31-65b5",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], "attentionHeadNames": ["L6H9", "L5H5"], "tokens": ["<|endoftext|>", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", ".", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", "."]}
     )
@@ -2086,9 +2086,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="8eb331ad-a881-4008-ab3d-cd2079daf729" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8eb331ad-a881-4008-ab3d-cd2079daf729")) {                    Plotly.newPlot(                        "8eb331ad-a881-4008-ab3d-cd2079daf729",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="8a472635-6d79-4701-854f-c62e62ebce54" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8a472635-6d79-4701-854f-c62e62ebce54")) {                    Plotly.newPlot(                        "8a472635-6d79-4701-854f-c62e62ebce54",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('8eb331ad-a881-4008-ab3d-cd2079daf729');
+var gd = document.getElementById('8a472635-6d79-4701-854f-c62e62ebce54');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2116,9 +2116,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="4e811a64-f177-4a07-a84b-b91d47b1f852" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4e811a64-f177-4a07-a84b-b91d47b1f852")) {                    Plotly.newPlot(                        "4e811a64-f177-4a07-a84b-b91d47b1f852",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="fae2e193-59a0-4811-84ef-98e7b34809aa" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("fae2e193-59a0-4811-84ef-98e7b34809aa")) {                    Plotly.newPlot(                        "fae2e193-59a0-4811-84ef-98e7b34809aa",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('4e811a64-f177-4a07-a84b-b91d47b1f852');
+var gd = document.getElementById('fae2e193-59a0-4811-84ef-98e7b34809aa');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2146,9 +2146,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="7a30321c-56de-47a4-81d8-46cfd886b36b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("7a30321c-56de-47a4-81d8-46cfd886b36b")) {                    Plotly.newPlot(                        "7a30321c-56de-47a4-81d8-46cfd886b36b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="ecf35438-ee23-4f6c-85d0-db50b0b55227" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ecf35438-ee23-4f6c-85d0-db50b0b55227")) {                    Plotly.newPlot(                        "ecf35438-ee23-4f6c-85d0-db50b0b55227",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('7a30321c-56de-47a4-81d8-46cfd886b36b');
+var gd = document.getElementById('ecf35438-ee23-4f6c-85d0-db50b0b55227');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2278,9 +2278,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="a3cc94cc-ab5d-48d5-afcc-c48d57f125ea" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a3cc94cc-ab5d-48d5-afcc-c48d57f125ea")) {                    Plotly.newPlot(                        "a3cc94cc-ab5d-48d5-afcc-c48d57f125ea",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="2ed18f74-fde1-4c81-ae6d-442fa345fd56" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2ed18f74-fde1-4c81-ae6d-442fa345fd56")) {                    Plotly.newPlot(                        "2ed18f74-fde1-4c81-ae6d-442fa345fd56",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('a3cc94cc-ab5d-48d5-afcc-c48d57f125ea');
+var gd = document.getElementById('2ed18f74-fde1-4c81-ae6d-442fa345fd56');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2308,9 +2308,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="38925f09-980d-4e2f-8ebb-727438003d35" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("38925f09-980d-4e2f-8ebb-727438003d35")) {                    Plotly.newPlot(                        "38925f09-980d-4e2f-8ebb-727438003d35",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],"xaxis":"x","y":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="69a381a6-98d5-41d7-b432-3b083c9cf66c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("69a381a6-98d5-41d7-b432-3b083c9cf66c")) {                    Plotly.newPlot(                        "69a381a6-98d5-41d7-b432-3b083c9cf66c",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],"xaxis":"x","y":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('38925f09-980d-4e2f-8ebb-727438003d35');
+var gd = document.getElementById('69a381a6-98d5-41d7-b432-3b083c9cf66c');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2388,7 +2388,7 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 </div>
 <p><strong>Exercise to the reader:</strong> Can you finish off this analysis? What’s going on here? Why are the backup name movers changing their behaviour? Why is one negative name mover becoming significantly less important?</p>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"44931b936f7e41a88aa39b6df566c649": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "68329682dcdf4df08289a498e7f1ca58": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "1c7bc6d96a044cb69b7e5e3dc190b2f3": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_44931b936f7e41a88aa39b6df566c649", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_68329682dcdf4df08289a498e7f1ca58", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "c14c9494085f4f6bb025489a0d6ccaed": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "88d1d898152b478c94604af43ac42d12": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "10327cf5d6bb477c9888e49d40d9b6f4": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c14c9494085f4f6bb025489a0d6ccaed", "placeholder": "\u200b", "style": "IPY_MODEL_88d1d898152b478c94604af43ac42d12", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "8446cb0777b7432c9f6f15127fae1ec1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "25355715e4f84b3fb30b7fb7d73dab5b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0542760fcf0c421e913bc2b839ac792c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8446cb0777b7432c9f6f15127fae1ec1", "placeholder": "\u200b", "style": "IPY_MODEL_25355715e4f84b3fb30b7fb7d73dab5b", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:02&lt;00:00,\u2007245MB/s]"}}, "0453b85ea3f4445e9c9a42ebae748343": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "adf78455ad4b4255b5ee12b111c8d6bf": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_10327cf5d6bb477c9888e49d40d9b6f4", "IPY_MODEL_1c7bc6d96a044cb69b7e5e3dc190b2f3", "IPY_MODEL_0542760fcf0c421e913bc2b839ac792c"], "layout": "IPY_MODEL_0453b85ea3f4445e9c9a42ebae748343", "tabbable": null, "tooltip": null}}, "4e0363da178140e38a1f5d5a763d8b9d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b071093461004612aea25c3faaa56d75": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "3328cd9e412b4e8ea246fbb324d064c2": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4e0363da178140e38a1f5d5a763d8b9d", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b071093461004612aea25c3faaa56d75", "tabbable": null, "tooltip": null, "value": 124.0}}, "cafce154ce254e2489b3c3231cd3afc8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "34dc2dd7d1b644148cf1162dd8fffae2": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "9987dd8fb4df47f8bad2480d55f0644e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_cafce154ce254e2489b3c3231cd3afc8", "placeholder": "\u200b", "style": "IPY_MODEL_34dc2dd7d1b644148cf1162dd8fffae2", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "36392e12f266416ea89267464302a9ae": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "39b9f02d659f4e00ae2652e1cd790372": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "5ce85280f03e416599efda8942bfd6b7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_36392e12f266416ea89267464302a9ae", "placeholder": "\u200b", "style": "IPY_MODEL_39b9f02d659f4e00ae2652e1cd790372", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.0kB/s]"}}, "77db344030f64e1cb37bde70ed8e7676": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f2a3e2ebfaad4d4d8b38096dac3f0a14": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_9987dd8fb4df47f8bad2480d55f0644e", "IPY_MODEL_3328cd9e412b4e8ea246fbb324d064c2", "IPY_MODEL_5ce85280f03e416599efda8942bfd6b7"], "layout": "IPY_MODEL_77db344030f64e1cb37bde70ed8e7676", "tabbable": null, "tooltip": null}}, "cb2bcca49ce84b5f813a3e09a28e17e8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7de92e4bb8af4843b82528edf005b6c3": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4ad663a579114d4eb47fd1364f66ccc0": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_cb2bcca49ce84b5f813a3e09a28e17e8", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_7de92e4bb8af4843b82528edf005b6c3", "tabbable": null, "tooltip": null, "value": 26.0}}, "8adc10691f3b407aa401a7cee384d572": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4dadf0b8acff4ab9a4d61fc31ce775ec": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "152fb19b656b444982eb11fae36392ae": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8adc10691f3b407aa401a7cee384d572", "placeholder": "\u200b", "style": "IPY_MODEL_4dadf0b8acff4ab9a4d61fc31ce775ec", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "822ddcdd61b443ce9bc565addafb6984": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "90da3925110a458aad1cc1c6f8f98fd4": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ef653dc2d7684ec5984b824ab60c017e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_822ddcdd61b443ce9bc565addafb6984", "placeholder": "\u200b", "style": "IPY_MODEL_90da3925110a458aad1cc1c6f8f98fd4", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.95kB/s]"}}, "729153cf51e244f598470b0f440f9b25": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ea70744b24bb4f8797907ee24a3e69c9": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_152fb19b656b444982eb11fae36392ae", "IPY_MODEL_4ad663a579114d4eb47fd1364f66ccc0", "IPY_MODEL_ef653dc2d7684ec5984b824ab60c017e"], "layout": "IPY_MODEL_729153cf51e244f598470b0f440f9b25", "tabbable": null, "tooltip": null}}, "4dc07cf1dbcb4f518a591b3bf3de91df": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ccd9adc54d384d26a6e04c69df253074": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "bba7279dfd574dd5b910973cde0fc709": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4dc07cf1dbcb4f518a591b3bf3de91df", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_ccd9adc54d384d26a6e04c69df253074", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "3b61ee1aea87467da549d7a70b026ce9": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5f44a32ffc6d4553ae2b471e7843b288": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "5604e63f816e45f280635edb76818789": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3b61ee1aea87467da549d7a70b026ce9", "placeholder": "\u200b", "style": "IPY_MODEL_5f44a32ffc6d4553ae2b471e7843b288", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "7f9b1c85b4da4b068ea5593efc94d962": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "68b77f75328e4a539b09b13b20cac43b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d8633f28258d46318664248a3566c7f3": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7f9b1c85b4da4b068ea5593efc94d962", "placeholder": "\u200b", "style": "IPY_MODEL_68b77f75328e4a539b09b13b20cac43b", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u20074.12MB/s]"}}, "4ab50487fcca41b59f1392f90cdc6757": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d682c8c5f56443c38680f259b4f0faef": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_5604e63f816e45f280635edb76818789", "IPY_MODEL_bba7279dfd574dd5b910973cde0fc709", "IPY_MODEL_d8633f28258d46318664248a3566c7f3"], "layout": "IPY_MODEL_4ab50487fcca41b59f1392f90cdc6757", "tabbable": null, "tooltip": null}}, "449a255401944642930f3f2821a91d6f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d096417d88bf4cc79562f5b69c150050": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a293e3738c334e4c9cc1bb9c71a81dc7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_449a255401944642930f3f2821a91d6f", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_d096417d88bf4cc79562f5b69c150050", "tabbable": null, "tooltip": null, "value": 456318.0}}, "877a40ace6ae4c65af3c6d4b089520da": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6147070de83046bca741d4491c2fbea6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "080b9c4c47814b0a82c5dea8974cab19": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_877a40ace6ae4c65af3c6d4b089520da", "placeholder": "\u200b", "style": "IPY_MODEL_6147070de83046bca741d4491c2fbea6", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "3261acc0fd974a1686a4fc899a6ba62f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "77914f1f07d648eab888c503c2a32cf0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8e014224685042fb868f27a39ba7175f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3261acc0fd974a1686a4fc899a6ba62f", "placeholder": "\u200b", "style": "IPY_MODEL_77914f1f07d648eab888c503c2a32cf0", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200729.0MB/s]"}}, "5b1acd258811455882cfb880a8d4790e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3790ba28df6f45f098a68fcec3f968f8": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_080b9c4c47814b0a82c5dea8974cab19", "IPY_MODEL_a293e3738c334e4c9cc1bb9c71a81dc7", "IPY_MODEL_8e014224685042fb868f27a39ba7175f"], "layout": "IPY_MODEL_5b1acd258811455882cfb880a8d4790e", "tabbable": null, "tooltip": null}}, "d811ce6fc2d94b5a8b4a8e4bb7b04432": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "372cd65f585d4674944843e298605bc0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "949ab7013dd74901865d601116408100": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d811ce6fc2d94b5a8b4a8e4bb7b04432", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_372cd65f585d4674944843e298605bc0", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "2fc1d8a4705f4cf180f6066cdb6000d7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7dbd77370d9a4010a60276a6071b73bd": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "24477b292cec43c38a49e1e7782f5827": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2fc1d8a4705f4cf180f6066cdb6000d7", "placeholder": "\u200b", "style": "IPY_MODEL_7dbd77370d9a4010a60276a6071b73bd", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "d310211006ce4cb6aac419247e69239f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "32671fce2b1942d8b7d158d4706c2713": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c1293200aaf4409a8b484e48a74056bb": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d310211006ce4cb6aac419247e69239f", "placeholder": "\u200b", "style": "IPY_MODEL_32671fce2b1942d8b7d158d4706c2713", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20074.54MB/s]"}}, "e79d7678437a45a39ca514276be1e4a9": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1fd27b66709b400bae1bba1dd1353b37": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_24477b292cec43c38a49e1e7782f5827", "IPY_MODEL_949ab7013dd74901865d601116408100", "IPY_MODEL_c1293200aaf4409a8b484e48a74056bb"], "layout": "IPY_MODEL_e79d7678437a45a39ca514276be1e4a9", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"fe14fba431ee44719e467a94d38602c1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "71480013d795482ea7827866fd7d1fb0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "2a992a646fbc405db6ec21ba17538e0e": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_fe14fba431ee44719e467a94d38602c1", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_71480013d795482ea7827866fd7d1fb0", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "b60c8dc4072a47f69bc60e5115ab87c6": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3dd3706dad6149a2ae71bfb8a4a5256a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0ccd5c3ba66d4687bda0b8a38511eecd": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b60c8dc4072a47f69bc60e5115ab87c6", "placeholder": "\u200b", "style": "IPY_MODEL_3dd3706dad6149a2ae71bfb8a4a5256a", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "1dffcf97acf041dfa4829930fc637e56": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bf988798a38d454fa0c118280dbeab56": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ddeeebc813c54775ab8841a88e512534": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_1dffcf97acf041dfa4829930fc637e56", "placeholder": "\u200b", "style": "IPY_MODEL_bf988798a38d454fa0c118280dbeab56", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:02&lt;00:00,\u2007243MB/s]"}}, "1acc38b256394079bf479fc9c3f6bbff": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2566b58370ab4ee1a23ec40b298e524d": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_0ccd5c3ba66d4687bda0b8a38511eecd", "IPY_MODEL_2a992a646fbc405db6ec21ba17538e0e", "IPY_MODEL_ddeeebc813c54775ab8841a88e512534"], "layout": "IPY_MODEL_1acc38b256394079bf479fc9c3f6bbff", "tabbable": null, "tooltip": null}}, "f06d8ed2f2fc40c78e6f71a980042fa2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0947963163124b84ad7bdbd0d14a5852": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "53f61b927c1346d88f4d1f2962f875ba": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f06d8ed2f2fc40c78e6f71a980042fa2", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_0947963163124b84ad7bdbd0d14a5852", "tabbable": null, "tooltip": null, "value": 124.0}}, "481fa4fd501940cebdf3f44be6c9e882": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ed04372d5ce8433d848d96e975723537": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "53bd3bcec1c747a0a2f91fd51bd8fba0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_481fa4fd501940cebdf3f44be6c9e882", "placeholder": "\u200b", "style": "IPY_MODEL_ed04372d5ce8433d848d96e975723537", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "4a23001ae4824747ac51dbe487a0a126": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "add5c8f3c40642ef8e4cf96579325ec3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c43b6d1b7dc04e8ea921c5940f890126": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4a23001ae4824747ac51dbe487a0a126", "placeholder": "\u200b", "style": "IPY_MODEL_add5c8f3c40642ef8e4cf96579325ec3", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200724.5kB/s]"}}, "b1a0d471042a4a018d4de3bf29a3ecd2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4c4a1ec2b7514c1297506faae67c1374": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_53bd3bcec1c747a0a2f91fd51bd8fba0", "IPY_MODEL_53f61b927c1346d88f4d1f2962f875ba", "IPY_MODEL_c43b6d1b7dc04e8ea921c5940f890126"], "layout": "IPY_MODEL_b1a0d471042a4a018d4de3bf29a3ecd2", "tabbable": null, "tooltip": null}}, "853700c97ecf475a8336c57945c87c6a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "47513102e87a47069219c229c4725eea": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "ba3727d9eb8e45b394738da6078d874c": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_853700c97ecf475a8336c57945c87c6a", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_47513102e87a47069219c229c4725eea", "tabbable": null, "tooltip": null, "value": 26.0}}, "3250ac219bb14a0d87380c723d7187c5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f6e7f731edb44410a1bbe854b20b75b3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "7daabc68020641e897d4809176829307": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3250ac219bb14a0d87380c723d7187c5", "placeholder": "\u200b", "style": "IPY_MODEL_f6e7f731edb44410a1bbe854b20b75b3", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "c0e90bf5be684b4eadbd6f97b968ab31": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e49babdcc96f4f99af8dba7cd03202fc": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ac60beb4421b4e758aec659715fa5146": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c0e90bf5be684b4eadbd6f97b968ab31", "placeholder": "\u200b", "style": "IPY_MODEL_e49babdcc96f4f99af8dba7cd03202fc", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.60kB/s]"}}, "97133900a1a94bc691ce291364fca64c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "54e358754fd241a7beb89dbd18d54469": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_7daabc68020641e897d4809176829307", "IPY_MODEL_ba3727d9eb8e45b394738da6078d874c", "IPY_MODEL_ac60beb4421b4e758aec659715fa5146"], "layout": "IPY_MODEL_97133900a1a94bc691ce291364fca64c", "tabbable": null, "tooltip": null}}, "488e5f543a8f48a79b17dd90afc28284": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7ffcda766f894799bb645944b5ecab6b": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "6b5428d552484be2b498aa06f2cd12d7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_488e5f543a8f48a79b17dd90afc28284", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_7ffcda766f894799bb645944b5ecab6b", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "9351dee6b05946a0a6880580b6f6ff6b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "924d96b2376c4b16a4541d64b0e45872": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "b9ffe90723d54a178c02c4a71079ef4f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9351dee6b05946a0a6880580b6f6ff6b", "placeholder": "\u200b", "style": "IPY_MODEL_924d96b2376c4b16a4541d64b0e45872", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "4647f9260ab248d7a350e7a07c999a9b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7f602aa1ed364df1a417f0cf429e4a5c": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d8504344e45a40dba0505fafd0a4fef8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4647f9260ab248d7a350e7a07c999a9b", "placeholder": "\u200b", "style": "IPY_MODEL_7f602aa1ed364df1a417f0cf429e4a5c", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200725.8MB/s]"}}, "c655320743a4421684aca235370c46c1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c9ac7e38eb534678896df289998d1855": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_b9ffe90723d54a178c02c4a71079ef4f", "IPY_MODEL_6b5428d552484be2b498aa06f2cd12d7", "IPY_MODEL_d8504344e45a40dba0505fafd0a4fef8"], "layout": "IPY_MODEL_c655320743a4421684aca235370c46c1", "tabbable": null, "tooltip": null}}, "4faadec4ac0f4a4ab32961fb3b74e529": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6ab9eaf5ee3342379119468f880a7e68": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "9a499b42e58849dcbfc030ebb0630a62": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4faadec4ac0f4a4ab32961fb3b74e529", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_6ab9eaf5ee3342379119468f880a7e68", "tabbable": null, "tooltip": null, "value": 456318.0}}, "621583ed869b478d913fc04a87073a95": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d2847e7beeb442018670534e35889459": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d3bf56862e814e46a08ce4949ce5df67": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_621583ed869b478d913fc04a87073a95", "placeholder": "\u200b", "style": "IPY_MODEL_d2847e7beeb442018670534e35889459", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "de2a35b1b3264fd2ac593af14da5b645": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c02c62e902e648ef9e245988ad7917f6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "b4faa038d86840faa63b4ff7f8ac404c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_de2a35b1b3264fd2ac593af14da5b645", "placeholder": "\u200b", "style": "IPY_MODEL_c02c62e902e648ef9e245988ad7917f6", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200738.7MB/s]"}}, "5fe7263a54b34f2b918ca2c57d194d74": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0b727299d6b94848ba14d79205e0d847": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d3bf56862e814e46a08ce4949ce5df67", "IPY_MODEL_9a499b42e58849dcbfc030ebb0630a62", "IPY_MODEL_b4faa038d86840faa63b4ff7f8ac404c"], "layout": "IPY_MODEL_5fe7263a54b34f2b918ca2c57d194d74", "tabbable": null, "tooltip": null}}, "860eac54afd644cdb66576f10d1c4b94": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b2dd202e54cc4c4bbf18779cef088e63": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "c9d427e21ae84197b7ff8cdb8bb4c89d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_860eac54afd644cdb66576f10d1c4b94", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b2dd202e54cc4c4bbf18779cef088e63", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "4973dab052f14376aa999fd6bc13197b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4064ed6431a14c8d940329269c2399e3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "aa33f2b3f7af4a5c956ebdbae08dabec": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4973dab052f14376aa999fd6bc13197b", "placeholder": "\u200b", "style": "IPY_MODEL_4064ed6431a14c8d940329269c2399e3", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "910235bb167a412ab2a54c930635b3bf": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f988b1609264444ea2e348b491a028a4": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0dc2fdfccac3440ea6a358b4b727da08": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_910235bb167a412ab2a54c930635b3bf", "placeholder": "\u200b", "style": "IPY_MODEL_f988b1609264444ea2e348b491a028a4", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u200747.4MB/s]"}}, "bec18b05c0a8413485c9f6fb6d8d51dd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "cf0868984e46408fb4a1d566ec3db0b0": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_aa33f2b3f7af4a5c956ebdbae08dabec", "IPY_MODEL_c9d427e21ae84197b7ff8cdb8bb4c89d", "IPY_MODEL_0dc2fdfccac3440ea6a358b4b727da08"], "layout": "IPY_MODEL_bec18b05c0a8413485c9f6fb6d8d51dd", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Exploratory_Analysis_Demo.ipynb b/generated/demos/Exploratory_Analysis_Demo.ipynb
index d4737e3db..8dbd19640 100644
--- a/generated/demos/Exploratory_Analysis_Demo.ipynb
+++ b/generated/demos/Exploratory_Analysis_Demo.ipynb
@@ -65,10 +65,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:03.791971Z",
-     "iopub.status.busy": "2024-11-19T14:46:03.791795Z",
-     "iopub.status.idle": "2024-11-19T14:46:03.801187Z",
-     "shell.execute_reply": "2024-11-19T14:46:03.800724Z"
+     "iopub.execute_input": "2024-12-14T00:56:47.879779Z",
+     "iopub.status.busy": "2024-12-14T00:56:47.879335Z",
+     "iopub.status.idle": "2024-12-14T00:56:47.889435Z",
+     "shell.execute_reply": "2024-12-14T00:56:47.888895Z"
     }
    },
    "outputs": [],
@@ -110,10 +110,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:03.803119Z",
-     "iopub.status.busy": "2024-11-19T14:46:03.802945Z",
-     "iopub.status.idle": "2024-11-19T14:46:06.974855Z",
-     "shell.execute_reply": "2024-11-19T14:46:06.974252Z"
+     "iopub.execute_input": "2024-12-14T00:56:47.891751Z",
+     "iopub.status.busy": "2024-12-14T00:56:47.891556Z",
+     "iopub.status.idle": "2024-12-14T00:56:51.109946Z",
+     "shell.execute_reply": "2024-12-14T00:56:51.109454Z"
     }
    },
    "outputs": [],
@@ -154,10 +154,10 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:06.977421Z",
-     "iopub.status.busy": "2024-11-19T14:46:06.977237Z",
-     "iopub.status.idle": "2024-11-19T14:46:06.980675Z",
-     "shell.execute_reply": "2024-11-19T14:46:06.980105Z"
+     "iopub.execute_input": "2024-12-14T00:56:51.112619Z",
+     "iopub.status.busy": "2024-12-14T00:56:51.112256Z",
+     "iopub.status.idle": "2024-12-14T00:56:51.115224Z",
+     "shell.execute_reply": "2024-12-14T00:56:51.114674Z"
     }
    },
    "outputs": [
@@ -193,10 +193,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:07.012999Z",
-     "iopub.status.busy": "2024-11-19T14:46:07.012758Z",
-     "iopub.status.idle": "2024-11-19T14:46:07.017020Z",
-     "shell.execute_reply": "2024-11-19T14:46:07.016553Z"
+     "iopub.execute_input": "2024-12-14T00:56:51.149823Z",
+     "iopub.status.busy": "2024-12-14T00:56:51.149327Z",
+     "iopub.status.idle": "2024-12-14T00:56:51.153801Z",
+     "shell.execute_reply": "2024-12-14T00:56:51.153349Z"
     }
    },
    "outputs": [],
@@ -281,17 +281,17 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:07.018918Z",
-     "iopub.status.busy": "2024-11-19T14:46:07.018749Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.075050Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.074481Z"
+     "iopub.execute_input": "2024-12-14T00:56:51.155889Z",
+     "iopub.status.busy": "2024-12-14T00:56:51.155539Z",
+     "iopub.status.idle": "2024-12-14T00:56:55.978791Z",
+     "shell.execute_reply": "2024-12-14T00:56:55.978209Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "adf78455ad4b4255b5ee12b111c8d6bf",
+       "model_id": "2566b58370ab4ee1a23ec40b298e524d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -305,7 +305,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f2a3e2ebfaad4d4d8b38096dac3f0a14",
+       "model_id": "4c4a1ec2b7514c1297506faae67c1374",
        "version_major": 2,
        "version_minor": 0
       },
@@ -319,7 +319,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ea70744b24bb4f8797907ee24a3e69c9",
+       "model_id": "54e358754fd241a7beb89dbd18d54469",
        "version_major": 2,
        "version_minor": 0
       },
@@ -333,7 +333,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d682c8c5f56443c38680f259b4f0faef",
+       "model_id": "c9ac7e38eb534678896df289998d1855",
        "version_major": 2,
        "version_minor": 0
       },
@@ -347,7 +347,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3790ba28df6f45f098a68fcec3f968f8",
+       "model_id": "0b727299d6b94848ba14d79205e0d847",
        "version_major": 2,
        "version_minor": 0
       },
@@ -361,7 +361,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1fd27b66709b400bae1bba1dd1353b37",
+       "model_id": "cf0868984e46408fb4a1d566ec3db0b0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -413,10 +413,10 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.077649Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.077161Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.408653Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.408162Z"
+     "iopub.execute_input": "2024-12-14T00:56:55.981331Z",
+     "iopub.status.busy": "2024-12-14T00:56:55.980946Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.260772Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.260166Z"
     }
    },
    "outputs": [
@@ -501,10 +501,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.410705Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.410529Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.417708Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.417256Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.263028Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.262828Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.270238Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.269664Z"
     }
    },
    "outputs": [
@@ -569,10 +569,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.419613Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.419427Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.425631Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.425113Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.272331Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.272032Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.278433Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.277895Z"
     }
    },
    "outputs": [
@@ -618,10 +618,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.427612Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.427300Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.697355Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.696699Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.280436Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.280126Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.563584Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.562963Z"
     }
    },
    "outputs": [],
@@ -644,10 +644,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.699901Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.699566Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.705116Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.704645Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.566041Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.565857Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.571671Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.571133Z"
     }
    },
    "outputs": [
@@ -788,10 +788,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.707060Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.706892Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.710687Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.710196Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.573659Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.573489Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.577324Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.576752Z"
     }
    },
    "outputs": [
@@ -839,10 +839,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.712880Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.712548Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.717405Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.716813Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.579546Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.579191Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.585331Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.584789Z"
     }
    },
    "outputs": [
@@ -895,10 +895,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.719435Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.719127Z",
-     "iopub.status.idle": "2024-11-19T14:46:13.722429Z",
-     "shell.execute_reply": "2024-11-19T14:46:13.721872Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.587226Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.587069Z",
+     "iopub.status.idle": "2024-12-14T00:56:56.590333Z",
+     "shell.execute_reply": "2024-12-14T00:56:56.589864Z"
     }
    },
    "outputs": [],
@@ -940,10 +940,10 @@
    "execution_count": 14,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:13.724579Z",
-     "iopub.status.busy": "2024-11-19T14:46:13.724168Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.465558Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.464897Z"
+     "iopub.execute_input": "2024-12-14T00:56:56.592375Z",
+     "iopub.status.busy": "2024-12-14T00:56:56.591968Z",
+     "iopub.status.idle": "2024-12-14T00:56:57.365301Z",
+     "shell.execute_reply": "2024-12-14T00:56:57.364657Z"
     }
    },
    "outputs": [
@@ -1929,9 +1929,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"efc11a98-bb03-49b5-8a98-2d19f7fe591b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"efc11a98-bb03-49b5-8a98-2d19f7fe591b\")) {                    Plotly.newPlot(                        \"efc11a98-bb03-49b5-8a98-2d19f7fe591b\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"49047634-c443-4608-b07a-015a3c6cf28b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"49047634-c443-4608-b07a-015a3c6cf28b\")) {                    Plotly.newPlot(                        \"49047634-c443-4608-b07a-015a3c6cf28b\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('efc11a98-bb03-49b5-8a98-2d19f7fe591b');\n",
+       "var gd = document.getElementById('49047634-c443-4608-b07a-015a3c6cf28b');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1996,10 +1996,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.503797Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.503437Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.537552Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.536987Z"
+     "iopub.execute_input": "2024-12-14T00:56:57.406173Z",
+     "iopub.status.busy": "2024-12-14T00:56:57.405744Z",
+     "iopub.status.idle": "2024-12-14T00:56:57.439909Z",
+     "shell.execute_reply": "2024-12-14T00:56:57.439283Z"
     }
    },
    "outputs": [
@@ -2959,9 +2959,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"9cfc6e76-339b-42f8-a6db-94e90576fc5e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"9cfc6e76-339b-42f8-a6db-94e90576fc5e\")) {                    Plotly.newPlot(                        \"9cfc6e76-339b-42f8-a6db-94e90576fc5e\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"619fc2fb-6643-474d-9104-a4fc84996d47\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"619fc2fb-6643-474d-9104-a4fc84996d47\")) {                    Plotly.newPlot(                        \"619fc2fb-6643-474d-9104-a4fc84996d47\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('9cfc6e76-339b-42f8-a6db-94e90576fc5e');\n",
+       "var gd = document.getElementById('619fc2fb-6643-474d-9104-a4fc84996d47');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3025,10 +3025,10 @@
    "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.539777Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.539436Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.594416Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.593813Z"
+     "iopub.execute_input": "2024-12-14T00:56:57.442269Z",
+     "iopub.status.busy": "2024-12-14T00:56:57.441922Z",
+     "iopub.status.idle": "2024-12-14T00:56:57.860668Z",
+     "shell.execute_reply": "2024-12-14T00:56:57.860098Z"
     }
    },
    "outputs": [
@@ -3055,172 +3055,172 @@
          "yaxis": "y",
          "z": [
           [
-           -0.0020563285797834396,
-           -0.0005101955030113459,
-           0.0004685768508352339,
-           0.00012511832755990326,
-           -0.0006028721109032631,
-           -0.00024295502225868404,
-           -0.002318894723430276,
-           -0.002758359769359231,
-           0.0005645868368446827,
-           0.000969740329310298,
-           -0.0002504501899238676,
-           4.740082658827305e-06
-          ],
-          [
-           -0.0010071131400763988,
-           0.0003947088844142854,
-           -0.0015487205237150192,
-           0.0014034901978448033,
-           -0.0012652688892558217,
-           -0.0011358250631019473,
-           -0.0028159404173493385,
-           -0.0029645359609276056,
+           -0.002056329045444727,
+           -0.0005102002760395408,
+           0.0004685759777203202,
+           0.00012511858949437737,
+           -0.0006028746138326824,
+           -0.00024295759794767946,
+           -0.0023188991472125053,
+           -0.0027583539485931396,
+           0.000564592657610774,
+           0.0009697366622276604,
+           -0.0002504411095287651,
+           4.7396752052009106e-06
+          ],
+          [
+           -0.0010071106953546405,
+           0.0003947066143155098,
+           -0.001548723317682743,
+           0.0014034844934940338,
+           -0.0012652697041630745,
+           -0.0011358254123479128,
+           -0.0028159399516880512,
+           -0.0029645331669598818,
            0.0029190238565206528,
            0.0025743518490344286,
            0.00036237656604498625,
-           0.0017548884497955441
-          ],
-          [
-           0.0005569332861341536,
-           -0.00112663593608886,
-           -0.0017354178708046675,
-           -0.0014514722861349583,
-           -0.00028740704874508083,
-           0.0017210595542564988,
-           0.00266590085811913,
-           0.0031146793626248837,
-           0.0005667305667884648,
-           -0.0036664949730038643,
-           -0.0018847067840397358,
-           7.027178071439266e-06
-          ],
-          [
-           -0.000726439815480262,
-           0.00011369686399120837,
-           0.001430142787285149,
-           0.000749052269384265,
-           0.0020184761378914118,
-           0.0007436758605763316,
-           -0.0004617759259417653,
-           -0.003905785735696554,
-           0.001140733016654849,
-           -4.027335671707988e-05,
-           -0.0013293256051838398,
-           -0.0017636881675571203
-          ],
-          [
-           -0.002828173339366913,
-           0.0003364472358953208,
-           -0.0014249038649722934,
-           -0.0037773081567138433,
-           0.0015998876187950373,
-           0.00029889732832089067,
-           -0.0008046309230849147,
-           0.0020388164557516575,
-           -0.0015593776479363441,
-           -0.0006437147385440767,
-           0.001116806990467012,
-           -0.00035003889934159815
-          ],
-          [
-           0.0011338151525706053,
-           0.0011259106686338782,
-           -0.0025163597892969847,
-           -0.0014790240675210953,
-           0.00038791983388364315,
-           -6.407807813957334e-05,
-           -0.0005096746608614922,
-           -0.0008841876406222582,
-           0.0006399309495463967,
-           -0.001009696745313704,
-           -0.0067590330727398396,
-           0.0033667273819446564
-          ],
-          [
-           -0.015147387981414795,
-           -0.002135086804628372,
-           0.002593189012259245,
-           -0.00042674108408391476,
-           -0.005559002980589867,
-           0.0026659294962882996,
-           0.006410874892026186,
-           -0.0038270100485533476,
-           -0.00038422830402851105,
-           -0.0016430210089311004,
-           -0.0013344308827072382,
-           -9.184109512716532e-05
-          ],
-          [
-           -9.488123760093004e-05,
-           -0.005788922309875488,
-           -0.0006383719155564904,
+           0.0017548904288560152
+          ],
+          [
+           0.0005569161148741841,
+           -0.0011266364017501473,
+           -0.0017354193842038512,
+           -0.0014514740323647857,
+           -0.0002874041674658656,
+           0.0017210585065186024,
+           0.0026659027207642794,
+           0.0031146786641329527,
+           0.0005667298682965338,
+           -0.003666497301310301,
+           -0.0018847138853743672,
+           7.028633262962103e-06
+          ],
+          [
+           -0.0007264401647262275,
+           0.0001136981591116637,
+           0.0014301439514383674,
+           0.0007490518037229776,
+           0.002018478699028492,
+           0.0007436765008606017,
+           -0.0004617785452865064,
+           -0.0039057875983417034,
+           0.0011407355777919292,
+           -4.027353134006262e-05,
+           -0.0013293151278048754,
+           -0.0017636835109442472
+          ],
+          [
+           -0.0028281747363507748,
+           0.0003364539588801563,
+           -0.0014249025844037533,
+           -0.003777292789891362,
+           0.001599886454641819,
+           0.00029889593133702874,
+           -0.0008046383736655116,
+           0.0020388178527355194,
+           -0.001559373107738793,
+           -0.0006437154370360076,
+           0.001116809668019414,
+           -0.0003500327584333718
+          ],
+          [
+           0.0011338141048327088,
+           0.0011259093880653381,
+           -0.0025163700338453054,
+           -0.0014790259301662445,
+           0.00038791849510744214,
+           -6.407558976206928e-05,
+           -0.0005096771637909114,
+           -0.0008841846138238907,
+           0.0006399258272722363,
+           -0.0010097046615555882,
+           -0.006759032607078552,
+           0.0033667325042188168
+          ],
+          [
+           -0.015147397294640541,
+           -0.0021350914612412453,
+           0.0025931934360414743,
+           -0.00042673421557992697,
+           -0.005559004843235016,
+           0.002665933221578598,
+           0.0064108846709132195,
+           -0.0038270088844001293,
+           -0.0003842375008389354,
+           -0.0016430213581770658,
+           -0.001334429020062089,
+           -9.183748625218868e-05
+          ],
+          [
+           -9.488424984738231e-05,
+           -0.005788922775536776,
+           -0.0006383699947036803,
            0.134933739900589,
-           -0.0017687628278508782,
+           -0.0017687629442662,
            -0.018917974084615707,
-           0.0038733629044145346,
-           -0.002145076170563698,
-           -0.010327237658202648,
-           0.18325874209403992,
-           -0.0007747883792035282,
-           -0.0010452116839587688
-          ],
-          [
-           -0.003833947703242302,
-           -0.000804627372417599,
-           -0.012673338875174522,
-           0.008045812137424946,
-           0.0036040153354406357,
-           -0.009398169815540314,
+           0.0038733729161322117,
+           -0.0021450738422572613,
+           -0.010327240452170372,
+           0.1832587718963623,
+           -0.0007747872150503099,
+           -0.0010452070273458958
+          ],
+          [
+           -0.0038339472375810146,
+           -0.0008046274306252599,
+           -0.01267334446310997,
+           0.008045826107263565,
+           0.003604009747505188,
+           -0.009398158639669418,
            -0.08272106945514679,
-           0.0035550352185964584,
-           -0.018404126167297363,
-           0.0017586719477549195,
-           0.2896132469177246,
-           0.022854045033454895
-          ],
-          [
-           0.08595201373100281,
-           -0.0006932567339390516,
-           0.06816966831684113,
-           0.01311141811311245,
-           -0.021098004654049873,
-           0.05112440511584282,
+           0.0035550370812416077,
+           -0.01840413361787796,
+           0.001758674974553287,
+           0.28961312770843506,
+           0.022854033857584
+          ],
+          [
+           0.08595199882984161,
+           -0.000693259877152741,
+           0.06816964596509933,
+           0.01311142835766077,
+           -0.02109798789024353,
+           0.05112443491816521,
            1.384489893913269,
-           0.04583733528852463,
-           -0.038303446024656296,
-           2.9854445457458496,
-           0.001966139767318964,
-           -0.008030213415622711
+           0.045837316662073135,
+           -0.038303449749946594,
+           2.9854443073272705,
+           0.001966138370335102,
+           -0.008030208759009838
           ],
           [
-           0.560872495174408,
+           0.5608724355697632,
            0.17083144187927246,
-           -0.033618733286857605,
-           0.05821547657251358,
-           -0.0024530075024813414,
-           0.0018771879840642214,
-           0.2882729768753052,
-           -1.898641586303711,
-           -0.001528693363070488,
-           -0.03513003885746002,
+           -0.033618729561567307,
+           0.05821548402309418,
+           -0.0024530175141990185,
+           0.0018771894974634051,
+           0.2882729470729828,
+           -1.8986413478851318,
+           -0.0015286938287317753,
+           -0.035130057483911514,
            0.48021769523620605,
-           -0.0009116916917264462
+           -0.0009116912842728198
           ],
           [
-           0.0160758625715971,
-           -0.03986112400889397,
+           0.016075868159532547,
+           -0.03986111283302307,
            -0.3879111707210541,
-           0.011123226955533028,
-           -0.005477802362293005,
-           -0.0025129495188593864,
-           -0.08056114614009857,
-           0.007518642581999302,
-           0.043011054396629333,
-           -0.0400824099779129,
-           -0.9702335596084595,
-           0.011862391605973244
+           0.01112320739775896,
+           -0.0054778107441961765,
+           -0.002512941136956215,
+           -0.08056112378835678,
+           0.007518645375967026,
+           0.04301108419895172,
+           -0.040082402527332306,
+           -0.9702335000038147,
+           0.011862380430102348
           ]
          ]
         }
@@ -4121,9 +4121,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d6688b13-6cd5-49b7-97f4-6d2139a939a2\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d6688b13-6cd5-49b7-97f4-6d2139a939a2\")) {                    Plotly.newPlot(                        \"d6688b13-6cd5-49b7-97f4-6d2139a939a2\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\")) {                    Plotly.newPlot(                        \"d13d12cd-e58c-4918-adf2-4cba0fcd4cfb\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06],[-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152],[0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06],[-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472],[-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718],[0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168],[-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05],[-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958],[-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584],[0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838],[0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198],[0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d6688b13-6cd5-49b7-97f4-6d2139a939a2');\n",
+       "var gd = document.getElementById('d13d12cd-e58c-4918-adf2-4cba0fcd4cfb');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -4195,10 +4195,10 @@
    "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.596489Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.596314Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.601764Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.601296Z"
+     "iopub.execute_input": "2024-12-14T00:56:57.863192Z",
+     "iopub.status.busy": "2024-12-14T00:56:57.862860Z",
+     "iopub.status.idle": "2024-12-14T00:56:57.868039Z",
+     "shell.execute_reply": "2024-12-14T00:56:57.867562Z"
     }
    },
    "outputs": [],
@@ -4263,29 +4263,29 @@
    "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.603608Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.603443Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.613813Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.613291Z"
+     "iopub.execute_input": "2024-12-14T00:56:57.870050Z",
+     "iopub.status.busy": "2024-12-14T00:56:57.869714Z",
+     "iopub.status.idle": "2024-12-14T00:56:57.879293Z",
+     "shell.execute_reply": "2024-12-14T00:56:57.878713Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-8c2f229e-31c6\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-238275f7-4d84\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-8c2f229e-31c6\",\n",
+       "      \"circuits-vis-238275f7-4d84\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], \"attentionHeadNames\": [\"L9H9\", \"L9H6\", \"L10H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-d75cc7e1-d317\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-a59ac87b-fe56\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-d75cc7e1-d317\",\n",
+       "      \"circuits-vis-a59ac87b-fe56\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L11H2\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -4414,10 +4414,10 @@
    "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.615803Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.615635Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.888838Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.888130Z"
+     "iopub.execute_input": "2024-12-14T00:56:57.881314Z",
+     "iopub.status.busy": "2024-12-14T00:56:57.881146Z",
+     "iopub.status.idle": "2024-12-14T00:56:58.167932Z",
+     "shell.execute_reply": "2024-12-14T00:56:58.167251Z"
     }
    },
    "outputs": [
@@ -4449,10 +4449,10 @@
    "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.891290Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.890913Z",
-     "iopub.status.idle": "2024-11-19T14:46:14.895755Z",
-     "shell.execute_reply": "2024-11-19T14:46:14.895210Z"
+     "iopub.execute_input": "2024-12-14T00:56:58.170608Z",
+     "iopub.status.busy": "2024-12-14T00:56:58.170212Z",
+     "iopub.status.idle": "2024-12-14T00:56:58.175091Z",
+     "shell.execute_reply": "2024-12-14T00:56:58.174594Z"
     }
    },
    "outputs": [
@@ -4493,10 +4493,10 @@
    "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:46:14.897950Z",
-     "iopub.status.busy": "2024-11-19T14:46:14.897780Z",
-     "iopub.status.idle": "2024-11-19T14:47:00.781210Z",
-     "shell.execute_reply": "2024-11-19T14:47:00.780619Z"
+     "iopub.execute_input": "2024-12-14T00:56:58.177256Z",
+     "iopub.status.busy": "2024-12-14T00:56:58.176915Z",
+     "iopub.status.idle": "2024-12-14T00:57:45.079131Z",
+     "shell.execute_reply": "2024-12-14T00:57:45.078479Z"
     }
    },
    "outputs": [],
@@ -4554,10 +4554,10 @@
    "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:47:00.783845Z",
-     "iopub.status.busy": "2024-11-19T14:47:00.783459Z",
-     "iopub.status.idle": "2024-11-19T14:47:00.815084Z",
-     "shell.execute_reply": "2024-11-19T14:47:00.814492Z"
+     "iopub.execute_input": "2024-12-14T00:57:45.081699Z",
+     "iopub.status.busy": "2024-12-14T00:57:45.081519Z",
+     "iopub.status.idle": "2024-12-14T00:57:45.113039Z",
+     "shell.execute_reply": "2024-12-14T00:57:45.112455Z"
     }
    },
    "outputs": [
@@ -5696,9 +5696,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d82fdf01-8df1-4af3-b10a-eb8e5065afdc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d82fdf01-8df1-4af3-b10a-eb8e5065afdc\")) {                    Plotly.newPlot(                        \"d82fdf01-8df1-4af3-b10a-eb8e5065afdc\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\")) {                    Plotly.newPlot(                        \"4bc9653c-0aeb-4930-9e54-0fc92a4e0591\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d82fdf01-8df1-4af3-b10a-eb8e5065afdc');\n",
+       "var gd = document.getElementById('4bc9653c-0aeb-4930-9e54-0fc92a4e0591');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -5758,10 +5758,10 @@
    "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:47:00.817052Z",
-     "iopub.status.busy": "2024-11-19T14:47:00.816866Z",
-     "iopub.status.idle": "2024-11-19T14:48:32.568738Z",
-     "shell.execute_reply": "2024-11-19T14:48:32.568173Z"
+     "iopub.execute_input": "2024-12-14T00:57:45.115204Z",
+     "iopub.status.busy": "2024-12-14T00:57:45.114883Z",
+     "iopub.status.idle": "2024-12-14T00:59:19.604005Z",
+     "shell.execute_reply": "2024-12-14T00:59:19.603315Z"
     }
    },
    "outputs": [],
@@ -5812,10 +5812,10 @@
    "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:48:32.571361Z",
-     "iopub.status.busy": "2024-11-19T14:48:32.570978Z",
-     "iopub.status.idle": "2024-11-19T14:48:32.601726Z",
-     "shell.execute_reply": "2024-11-19T14:48:32.601137Z"
+     "iopub.execute_input": "2024-12-14T00:59:19.606789Z",
+     "iopub.status.busy": "2024-12-14T00:59:19.606567Z",
+     "iopub.status.idle": "2024-12-14T00:59:19.637315Z",
+     "shell.execute_reply": "2024-12-14T00:59:19.636727Z"
     }
    },
    "outputs": [
@@ -6954,9 +6954,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"e0f8d107-6766-415c-8f1a-09e327e99d13\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e0f8d107-6766-415c-8f1a-09e327e99d13\")) {                    Plotly.newPlot(                        \"e0f8d107-6766-415c-8f1a-09e327e99d13\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"191ca322-bcdc-49ad-a055-e5a33b6bafa3\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"191ca322-bcdc-49ad-a055-e5a33b6bafa3\")) {                    Plotly.newPlot(                        \"191ca322-bcdc-49ad-a055-e5a33b6bafa3\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e0f8d107-6766-415c-8f1a-09e327e99d13');\n",
+       "var gd = document.getElementById('191ca322-bcdc-49ad-a055-e5a33b6bafa3');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -7018,10 +7018,10 @@
    "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:48:32.603820Z",
-     "iopub.status.busy": "2024-11-19T14:48:32.603488Z",
-     "iopub.status.idle": "2024-11-19T14:48:32.631949Z",
-     "shell.execute_reply": "2024-11-19T14:48:32.631426Z"
+     "iopub.execute_input": "2024-12-14T00:59:19.639547Z",
+     "iopub.status.busy": "2024-12-14T00:59:19.639233Z",
+     "iopub.status.idle": "2024-12-14T00:59:19.670098Z",
+     "shell.execute_reply": "2024-12-14T00:59:19.669543Z"
     }
    },
    "outputs": [
@@ -8160,9 +8160,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"611a846b-5fc0-4193-8bf8-9009ceffb348\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"611a846b-5fc0-4193-8bf8-9009ceffb348\")) {                    Plotly.newPlot(                        \"611a846b-5fc0-4193-8bf8-9009ceffb348\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"53f8687c-322d-426d-8a84-d3e680bc0251\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"53f8687c-322d-426d-8a84-d3e680bc0251\")) {                    Plotly.newPlot(                        \"53f8687c-322d-426d-8a84-d3e680bc0251\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('611a846b-5fc0-4193-8bf8-9009ceffb348');\n",
+       "var gd = document.getElementById('53f8687c-322d-426d-8a84-d3e680bc0251');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -8216,10 +8216,10 @@
    "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:48:32.634103Z",
-     "iopub.status.busy": "2024-11-19T14:48:32.633781Z",
-     "iopub.status.idle": "2024-11-19T14:49:09.327785Z",
-     "shell.execute_reply": "2024-11-19T14:49:09.327149Z"
+     "iopub.execute_input": "2024-12-14T00:59:19.672437Z",
+     "iopub.status.busy": "2024-12-14T00:59:19.672240Z",
+     "iopub.status.idle": "2024-12-14T00:59:57.559193Z",
+     "shell.execute_reply": "2024-12-14T00:59:57.558632Z"
     }
    },
    "outputs": [],
@@ -8266,10 +8266,10 @@
    "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:49:09.330497Z",
-     "iopub.status.busy": "2024-11-19T14:49:09.330141Z",
-     "iopub.status.idle": "2024-11-19T14:49:09.360253Z",
-     "shell.execute_reply": "2024-11-19T14:49:09.359680Z"
+     "iopub.execute_input": "2024-12-14T00:59:57.561760Z",
+     "iopub.status.busy": "2024-12-14T00:59:57.561408Z",
+     "iopub.status.idle": "2024-12-14T00:59:57.591996Z",
+     "shell.execute_reply": "2024-12-14T00:59:57.591441Z"
     }
    },
    "outputs": [
@@ -9355,9 +9355,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"f5742c28-dd8a-48dc-a7e5-91b761e718a8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f5742c28-dd8a-48dc-a7e5-91b761e718a8\")) {                    Plotly.newPlot(                        \"f5742c28-dd8a-48dc-a7e5-91b761e718a8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"d7db9d56-c782-4460-85c8-5165d9b90be4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d7db9d56-c782-4460-85c8-5165d9b90be4\")) {                    Plotly.newPlot(                        \"d7db9d56-c782-4460-85c8-5165d9b90be4\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f5742c28-dd8a-48dc-a7e5-91b761e718a8');\n",
+       "var gd = document.getElementById('d7db9d56-c782-4460-85c8-5165d9b90be4');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -9420,10 +9420,10 @@
    "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:49:09.362280Z",
-     "iopub.status.busy": "2024-11-19T14:49:09.362108Z",
-     "iopub.status.idle": "2024-11-19T14:49:46.008399Z",
-     "shell.execute_reply": "2024-11-19T14:49:46.007830Z"
+     "iopub.execute_input": "2024-12-14T00:59:57.594318Z",
+     "iopub.status.busy": "2024-12-14T00:59:57.594131Z",
+     "iopub.status.idle": "2024-12-14T01:00:35.562582Z",
+     "shell.execute_reply": "2024-12-14T01:00:35.562008Z"
     }
    },
    "outputs": [],
@@ -9458,10 +9458,10 @@
    "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:49:46.010935Z",
-     "iopub.status.busy": "2024-11-19T14:49:46.010760Z",
-     "iopub.status.idle": "2024-11-19T14:49:46.042201Z",
-     "shell.execute_reply": "2024-11-19T14:49:46.041771Z"
+     "iopub.execute_input": "2024-12-14T01:00:35.565275Z",
+     "iopub.status.busy": "2024-12-14T01:00:35.564875Z",
+     "iopub.status.idle": "2024-12-14T01:00:35.595964Z",
+     "shell.execute_reply": "2024-12-14T01:00:35.595316Z"
     }
    },
    "outputs": [
@@ -10547,9 +10547,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"41ce7e50-d3c8-4c1e-9792-e60a432c716e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"41ce7e50-d3c8-4c1e-9792-e60a432c716e\")) {                    Plotly.newPlot(                        \"41ce7e50-d3c8-4c1e-9792-e60a432c716e\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"b78bb017-f4f8-4418-b905-eac4d9764ea8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b78bb017-f4f8-4418-b905-eac4d9764ea8\")) {                    Plotly.newPlot(                        \"b78bb017-f4f8-4418-b905-eac4d9764ea8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('41ce7e50-d3c8-4c1e-9792-e60a432c716e');\n",
+       "var gd = document.getElementById('b78bb017-f4f8-4418-b905-eac4d9764ea8');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -10600,10 +10600,10 @@
    "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:49:46.044469Z",
-     "iopub.status.busy": "2024-11-19T14:49:46.044066Z",
-     "iopub.status.idle": "2024-11-19T14:49:46.087495Z",
-     "shell.execute_reply": "2024-11-19T14:49:46.086991Z"
+     "iopub.execute_input": "2024-12-14T01:00:35.598084Z",
+     "iopub.status.busy": "2024-12-14T01:00:35.597908Z",
+     "iopub.status.idle": "2024-12-14T01:00:35.643804Z",
+     "shell.execute_reply": "2024-12-14T01:00:35.643178Z"
     }
    },
    "outputs": [
@@ -12117,9 +12117,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"524094c7-fa65-4920-a689-538877ae3454\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"524094c7-fa65-4920-a689-538877ae3454\")) {                    Plotly.newPlot(                        \"524094c7-fa65-4920-a689-538877ae3454\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"c2250b6e-3006-4a68-a458-2890b3c0d4df\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c2250b6e-3006-4a68-a458-2890b3c0d4df\")) {                    Plotly.newPlot(                        \"c2250b6e-3006-4a68-a458-2890b3c0d4df\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('524094c7-fa65-4920-a689-538877ae3454');\n",
+       "var gd = document.getElementById('c2250b6e-3006-4a68-a458-2890b3c0d4df');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -12181,10 +12181,10 @@
    "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:49:46.089721Z",
-     "iopub.status.busy": "2024-11-19T14:49:46.089550Z",
-     "iopub.status.idle": "2024-11-19T14:50:23.429796Z",
-     "shell.execute_reply": "2024-11-19T14:50:23.429141Z"
+     "iopub.execute_input": "2024-12-14T01:00:35.646202Z",
+     "iopub.status.busy": "2024-12-14T01:00:35.645977Z",
+     "iopub.status.idle": "2024-12-14T01:01:13.540641Z",
+     "shell.execute_reply": "2024-12-14T01:01:13.539993Z"
     }
    },
    "outputs": [],
@@ -12224,10 +12224,10 @@
    "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:23.432549Z",
-     "iopub.status.busy": "2024-11-19T14:50:23.432318Z",
-     "iopub.status.idle": "2024-11-19T14:50:23.491007Z",
-     "shell.execute_reply": "2024-11-19T14:50:23.490413Z"
+     "iopub.execute_input": "2024-12-14T01:01:13.543284Z",
+     "iopub.status.busy": "2024-12-14T01:01:13.543099Z",
+     "iopub.status.idle": "2024-12-14T01:01:13.603349Z",
+     "shell.execute_reply": "2024-12-14T01:01:13.602719Z"
     }
    },
    "outputs": [
@@ -13313,9 +13313,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"06bfd2af-c00f-4a8b-93e6-4e78e590b1b5\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"06bfd2af-c00f-4a8b-93e6-4e78e590b1b5\")) {                    Plotly.newPlot(                        \"06bfd2af-c00f-4a8b-93e6-4e78e590b1b5\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"f21b691b-6def-4049-8b21-6c417fc5a93c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f21b691b-6def-4049-8b21-6c417fc5a93c\")) {                    Plotly.newPlot(                        \"f21b691b-6def-4049-8b21-6c417fc5a93c\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('06bfd2af-c00f-4a8b-93e6-4e78e590b1b5');\n",
+       "var gd = document.getElementById('f21b691b-6def-4049-8b21-6c417fc5a93c');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14650,9 +14650,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"c933b3d0-ef0f-4d9d-8702-95ead096afef\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c933b3d0-ef0f-4d9d-8702-95ead096afef\")) {                    Plotly.newPlot(                        \"c933b3d0-ef0f-4d9d-8702-95ead096afef\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"d6f23061-00f8-4a84-9d17-cede03fea051\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d6f23061-00f8-4a84-9d17-cede03fea051\")) {                    Plotly.newPlot(                        \"d6f23061-00f8-4a84-9d17-cede03fea051\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('c933b3d0-ef0f-4d9d-8702-95ead096afef');\n",
+       "var gd = document.getElementById('d6f23061-00f8-4a84-9d17-cede03fea051');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14731,37 +14731,37 @@
    "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:23.493043Z",
-     "iopub.status.busy": "2024-11-19T14:50:23.492847Z",
-     "iopub.status.idle": "2024-11-19T14:50:23.505856Z",
-     "shell.execute_reply": "2024-11-19T14:50:23.505347Z"
+     "iopub.execute_input": "2024-12-14T01:01:13.605613Z",
+     "iopub.status.busy": "2024-12-14T01:01:13.605390Z",
+     "iopub.status.idle": "2024-12-14T01:01:13.618831Z",
+     "shell.execute_reply": "2024-12-14T01:01:13.618283Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-1a7d2574-bd46\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-cec38c2d-008b\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-1a7d2574-bd46\",\n",
+       "      \"circuits-vis-cec38c2d-008b\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], \"attentionHeadNames\": [\"L5H5\", \"L6H9\", \"L3H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-54ca365f-dfb7\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-4796b044-6a22\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-54ca365f-dfb7\",\n",
+       "      \"circuits-vis-4796b044-6a22\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], \"attentionHeadNames\": [\"L8H6\", \"L8H10\", \"L7H9\", \"L7H3\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-40e7b9a3-aa19\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-3f1d1fde-8de5\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-40e7b9a3-aa19\",\n",
+       "      \"circuits-vis-3f1d1fde-8de5\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L9H9\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -14876,10 +14876,10 @@
    "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:23.508122Z",
-     "iopub.status.busy": "2024-11-19T14:50:23.507727Z",
-     "iopub.status.idle": "2024-11-19T14:50:23.666244Z",
-     "shell.execute_reply": "2024-11-19T14:50:23.665676Z"
+     "iopub.execute_input": "2024-12-14T01:01:13.621272Z",
+     "iopub.status.busy": "2024-12-14T01:01:13.620857Z",
+     "iopub.status.idle": "2024-12-14T01:01:13.781947Z",
+     "shell.execute_reply": "2024-12-14T01:01:13.781299Z"
     }
    },
    "outputs": [],
@@ -14898,21 +14898,21 @@
    "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:23.668861Z",
-     "iopub.status.busy": "2024-11-19T14:50:23.668647Z",
-     "iopub.status.idle": "2024-11-19T14:50:23.678546Z",
-     "shell.execute_reply": "2024-11-19T14:50:23.678063Z"
+     "iopub.execute_input": "2024-12-14T01:01:13.784795Z",
+     "iopub.status.busy": "2024-12-14T01:01:13.784431Z",
+     "iopub.status.idle": "2024-12-14T01:01:13.794420Z",
+     "shell.execute_reply": "2024-12-14T01:01:13.793896Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-c41df250-c3be\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-2ebcde31-65b5\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-c41df250-c3be\",\n",
+       "      \"circuits-vis-2ebcde31-65b5\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], \"attentionHeadNames\": [\"L6H9\", \"L5H5\"], \"tokens\": [\"<|endoftext|>\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\"]}\n",
        "    )\n",
@@ -15001,10 +15001,10 @@
    "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:23.680669Z",
-     "iopub.status.busy": "2024-11-19T14:50:23.680338Z",
-     "iopub.status.idle": "2024-11-19T14:50:24.451257Z",
-     "shell.execute_reply": "2024-11-19T14:50:24.450662Z"
+     "iopub.execute_input": "2024-12-14T01:01:13.796662Z",
+     "iopub.status.busy": "2024-12-14T01:01:13.796322Z",
+     "iopub.status.idle": "2024-12-14T01:01:14.592546Z",
+     "shell.execute_reply": "2024-12-14T01:01:14.592025Z"
     }
    },
    "outputs": [
@@ -15102,10 +15102,10 @@
    "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:24.453623Z",
-     "iopub.status.busy": "2024-11-19T14:50:24.453283Z",
-     "iopub.status.idle": "2024-11-19T14:50:24.533874Z",
-     "shell.execute_reply": "2024-11-19T14:50:24.533293Z"
+     "iopub.execute_input": "2024-12-14T01:01:14.594877Z",
+     "iopub.status.busy": "2024-12-14T01:01:14.594530Z",
+     "iopub.status.idle": "2024-12-14T01:01:14.678853Z",
+     "shell.execute_reply": "2024-12-14T01:01:14.678279Z"
     }
    },
    "outputs": [
@@ -16191,9 +16191,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"8eb331ad-a881-4008-ab3d-cd2079daf729\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8eb331ad-a881-4008-ab3d-cd2079daf729\")) {                    Plotly.newPlot(                        \"8eb331ad-a881-4008-ab3d-cd2079daf729\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"8a472635-6d79-4701-854f-c62e62ebce54\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8a472635-6d79-4701-854f-c62e62ebce54\")) {                    Plotly.newPlot(                        \"8a472635-6d79-4701-854f-c62e62ebce54\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('8eb331ad-a881-4008-ab3d-cd2079daf729');\n",
+       "var gd = document.getElementById('8a472635-6d79-4701-854f-c62e62ebce54');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -17303,9 +17303,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"4e811a64-f177-4a07-a84b-b91d47b1f852\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4e811a64-f177-4a07-a84b-b91d47b1f852\")) {                    Plotly.newPlot(                        \"4e811a64-f177-4a07-a84b-b91d47b1f852\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"fae2e193-59a0-4811-84ef-98e7b34809aa\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"fae2e193-59a0-4811-84ef-98e7b34809aa\")) {                    Plotly.newPlot(                        \"fae2e193-59a0-4811-84ef-98e7b34809aa\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('4e811a64-f177-4a07-a84b-b91d47b1f852');\n",
+       "var gd = document.getElementById('fae2e193-59a0-4811-84ef-98e7b34809aa');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18415,9 +18415,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"7a30321c-56de-47a4-81d8-46cfd886b36b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"7a30321c-56de-47a4-81d8-46cfd886b36b\")) {                    Plotly.newPlot(                        \"7a30321c-56de-47a4-81d8-46cfd886b36b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"ecf35438-ee23-4f6c-85d0-db50b0b55227\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ecf35438-ee23-4f6c-85d0-db50b0b55227\")) {                    Plotly.newPlot(                        \"ecf35438-ee23-4f6c-85d0-db50b0b55227\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('7a30321c-56de-47a4-81d8-46cfd886b36b');\n",
+       "var gd = document.getElementById('ecf35438-ee23-4f6c-85d0-db50b0b55227');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18509,10 +18509,10 @@
    "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:24.536020Z",
-     "iopub.status.busy": "2024-11-19T14:50:24.535685Z",
-     "iopub.status.idle": "2024-11-19T14:50:24.824052Z",
-     "shell.execute_reply": "2024-11-19T14:50:24.823444Z"
+     "iopub.execute_input": "2024-12-14T01:01:14.681801Z",
+     "iopub.status.busy": "2024-12-14T01:01:14.681439Z",
+     "iopub.status.idle": "2024-12-14T01:01:14.974170Z",
+     "shell.execute_reply": "2024-12-14T01:01:14.973548Z"
     }
    },
    "outputs": [
@@ -18577,10 +18577,10 @@
    "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:24.826533Z",
-     "iopub.status.busy": "2024-11-19T14:50:24.826202Z",
-     "iopub.status.idle": "2024-11-19T14:50:24.908719Z",
-     "shell.execute_reply": "2024-11-19T14:50:24.908135Z"
+     "iopub.execute_input": "2024-12-14T01:01:14.976750Z",
+     "iopub.status.busy": "2024-12-14T01:01:14.976358Z",
+     "iopub.status.idle": "2024-12-14T01:01:15.444633Z",
+     "shell.execute_reply": "2024-12-14T01:01:15.444055Z"
     }
    },
    "outputs": [
@@ -18607,172 +18607,172 @@
          "yaxis": "y",
          "z": [
           [
-           -0.002156492555513978,
-           -0.00046507816296070814,
-           0.00024167407536879182,
-           0.00028066313825547695,
-           -0.00041629592305980623,
-           -0.0004892397555522621,
-           -0.0026209561619907618,
-           -0.0029356726445257664,
-           0.000425610167440027,
-           0.0005418087239377201,
-           0.0002375440817559138,
-           -7.489288691431284e-05
-          ],
-          [
-           -0.0006585267838090658,
-           0.00040605987305752933,
-           -0.0009330196771770716,
-           0.0008937990060076118,
-           -0.0009785185102373362,
-           -0.0005338399787433445,
-           -0.0027988061774522066,
-           -0.004214102402329445,
-           0.0025785816833376884,
-           0.002450671512633562,
-           0.0005351606523618102,
-           0.0012349870521575212
-          ],
-          [
-           0.0009405002347193658,
-           -0.0011168736964464188,
-           -0.0011542305583134294,
-           -0.0015697323251515627,
-           -0.0005699749453924596,
-           0.0014514662325382233,
-           0.0024399266112595797,
-           0.0031583067029714584,
-           0.0009236747864633799,
-           -0.00357811083085835,
-           -0.0010650111362338066,
-           -0.000355880125425756
-          ],
-          [
-           -0.0005624890327453613,
-           -1.1902564438059926e-05,
-           0.0011530898045748472,
-           0.0007360299350693822,
-           0.0016493811272084713,
-           0.0008800593786872923,
-           -0.0006905524642206728,
-           -0.0030319970101118088,
-           0.0008080819388851523,
-           0.00010364333866164088,
-           -0.0005807871930301189,
-           -0.0011067159939557314
-          ],
-          [
-           -0.0026376438327133656,
-           0.00026929602609016,
-           -0.001641781534999609,
-           -0.0034070422407239676,
-           0.001744971377775073,
-           0.00046448659850284457,
-           -0.000789930927567184,
-           0.0018328832229599357,
-           -0.0008632286335341632,
-           -0.00039789103902876377,
-           0.000787912926170975,
-           -0.00012039250577799976
-          ],
-          [
-           0.0008688485249876976,
-           0.0009473453392274678,
-           -0.0022809950169175863,
-           -0.0011803111992776394,
-           0.00024082366144284606,
-           -0.0004318496794439852,
-           -0.0003728233277797699,
-           -0.0007385354256257415,
-           0.0008114463416859508,
-           -0.00040441250894218683,
-           -0.007074240129441023,
-           0.003946419805288315
-          ],
-          [
-           -0.01491759717464447,
-           -0.002280194777995348,
-           0.0022679578978568316,
-           -8.297746535390615e-05,
-           -0.004981024190783501,
-           0.0027670864947140217,
-           0.00626616645604372,
-           -0.0034861797466874123,
-           -0.0013347704662010074,
-           -0.0017918730154633522,
-           -0.001223177881911397,
-           0.000405364902690053
-          ],
-          [
-           -0.00024617952294647694,
-           -0.0057901544496417046,
-           -0.0004975934862159193,
-           0.14218278229236603,
-           -0.001496216980740428,
-           -0.01900637149810791,
-           0.0031333654187619686,
-           -0.0018582025077193975,
+           -0.002156495349481702,
+           -0.00046507216757163405,
+           0.00024167195078916848,
+           0.0002806607517413795,
+           -0.0004162961558904499,
+           -0.0004892406286671758,
+           -0.0026209622155874968,
+           -0.0029356717132031918,
+           0.0004256087704561651,
+           0.0005418135551735759,
+           0.00023754549329169095,
+           -7.4885378126055e-05
+          ],
+          [
+           -0.0006585237570106983,
+           0.00040605897083878517,
+           -0.0009330313769169152,
+           0.0008937875390984118,
+           -0.0009785089641809464,
+           -0.0005338447517715394,
+           -0.002798808738589287,
+           -0.0042141154408454895,
+           0.0025785760954022408,
+           0.002450673608109355,
+           0.0005351657164283097,
+           0.0012349945027381182
+          ],
+          [
+           0.0009405009332112968,
+           -0.0011168718338012695,
+           -0.0011542299762368202,
+           -0.001569736865349114,
+           -0.0005699718603864312,
+           0.0014514633221551776,
+           0.0024399259127676487,
+           0.003158304840326309,
+           0.0009236858459189534,
+           -0.0035781036131083965,
+           -0.001065023592673242,
+           -0.00035587642923928797
+          ],
+          [
+           -0.0005624899640679359,
+           -1.1904543498530984e-05,
+           0.0011530885240063071,
+           0.0007360292365774512,
+           0.0016493821749463677,
+           0.0008800605428405106,
+           -0.0006905477494001389,
+           -0.0030319932848215103,
+           0.0008080820553004742,
+           0.00010364956688135862,
+           -0.000580791849642992,
+           -0.0011067147133871913
+          ],
+          [
+           -0.0026376410387456417,
+           0.0002693022252060473,
+           -0.0016417787410318851,
+           -0.0034070289693772793,
+           0.0017449733568355441,
+           0.00046448662760667503,
+           -0.0007899348856881261,
+           0.0018328834557905793,
+           -0.0008632306708022952,
+           -0.00039788917638361454,
+           0.0007879154873080552,
+           -0.00012039285502396524
+          ],
+          [
+           0.0008688486414030194,
+           0.0009473506361246109,
+           -0.002280982444062829,
+           -0.0011803142260760069,
+           0.0002408194704912603,
+           -0.0004318461287766695,
+           -0.00037281878758221865,
+           -0.0007385292556136847,
+           0.0008114340016618371,
+           -0.00040442857425659895,
+           -0.007074241526424885,
+           0.003946421667933464
+          ],
+          [
+           -0.014917591586709023,
+           -0.0022801952436566353,
+           0.0022679539397358894,
+           -8.297240128740668e-05,
+           -0.0049810330383479595,
+           0.0027670827694237232,
+           0.0062661729753017426,
+           -0.0034861767198890448,
+           -0.0013347702333703637,
+           -0.001791873830370605,
+           -0.0012231743894517422,
+           0.0004053567536175251
+          ],
+          [
+           -0.0002461824333295226,
+           -0.005790156312286854,
+           -0.0004975919146090746,
+           0.14218276739120483,
+           -0.0014962118584662676,
+           -0.01900637522339821,
+           0.003133367281407118,
+           -0.0018581977346912026,
            -0.011305071413516998,
-           0.19225990772247314,
-           -0.0011892484035342932,
-           -0.0010282367002218962
-          ],
-          [
-           -0.0038004019297659397,
-           -0.0008571128128096461,
-           -0.013956686481833458,
-           0.008289236575365067,
-           0.004314992111176252,
-           -0.009073692373931408,
-           -0.08315175771713257,
-           0.0034568109549582005,
-           -0.018055014312267303,
-           0.00217800703831017,
+           0.19225995242595673,
+           -0.0011892454931512475,
+           -0.0010282358853146434
+          ],
+          [
+           -0.0038004028610885143,
+           -0.0008571104845032096,
+           -0.013956702314317226,
+           0.008289228193461895,
+           0.004315000958740711,
+           -0.009073707275092602,
+           -0.08315176516771317,
+           0.003456807229667902,
+           -0.01805501990020275,
+           0.00217801658436656,
            0.29780468344688416,
-           0.02409377135336399
+           0.024093760177493095
           ],
           [
            0.08904657512903214,
-           -0.0007931506261229515,
-           0.07247606664896011,
-           0.015016509220004082,
-           -0.021209245547652245,
-           0.052054572850465775,
+           -0.0007931562722660601,
+           0.07247605919837952,
+           0.01501650083810091,
+           -0.021209243685007095,
+           0.05205458402633667,
            1.441115140914917,
            0.04743725806474686,
            -0.0322909839451313,
            0.0,
-           0.0019993034657090902,
-           -0.008072325959801674
+           0.0019993027672171593,
+           -0.008072329685091972
           ],
           [
-           0.8600812554359436,
-           0.3260071277618408,
+           0.8600811958312988,
+           0.32600724697113037,
            0.16344299912452698,
-           0.07133564352989197,
+           0.07133562117815018,
            -0.004448350518941879,
-           0.0006813818472437561,
-           0.36613476276397705,
-           -0.710505485534668,
-           -0.0020313779823482037,
-           -0.032143473625183105,
-           1.2294319868087769,
-           0.0018451482756063342
-          ],
-          [
-           0.016877301037311554,
-           -0.001730009913444519,
-           -0.501085638999939,
-           0.02749774605035782,
-           -0.005966278724372387,
-           -0.0049441163428127766,
+           0.0006813746877014637,
+           0.3661348223686218,
+           -0.7105053663253784,
+           -0.0020313761197030544,
+           -0.03214346989989281,
+           1.2294316291809082,
+           0.0018451516516506672
+          ],
+          [
+           0.016877306625247,
+           -0.0017300043255090714,
+           -0.5010855793952942,
+           0.02749773859977722,
+           -0.005966317839920521,
+           -0.004944117274135351,
            -0.08855222165584564,
-           0.006622320972383022,
-           0.04412432014942169,
-           -0.027266837656497955,
+           0.006622340530157089,
+           0.04412432760000229,
+           -0.02726682648062706,
            -1.1349124908447266,
-           0.02287965640425682
+           0.022879652678966522
           ]
          ]
         }
@@ -19673,9 +19673,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"a3cc94cc-ab5d-48d5-afcc-c48d57f125ea\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a3cc94cc-ab5d-48d5-afcc-c48d57f125ea\")) {                    Plotly.newPlot(                        \"a3cc94cc-ab5d-48d5-afcc-c48d57f125ea\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"2ed18f74-fde1-4c81-ae6d-442fa345fd56\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2ed18f74-fde1-4c81-ae6d-442fa345fd56\")) {                    Plotly.newPlot(                        \"2ed18f74-fde1-4c81-ae6d-442fa345fd56\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05],[-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182],[0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797],[-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913],[-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524],[0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464],[-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251],[-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434],[-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095],[0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972],[0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672],[0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('a3cc94cc-ab5d-48d5-afcc-c48d57f125ea');\n",
+       "var gd = document.getElementById('2ed18f74-fde1-4c81-ae6d-442fa345fd56');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -19869,297 +19869,297 @@
          "showlegend": false,
          "type": "scatter",
          "x": [
-          -0.002156492555513978,
-          -0.00046507816296070814,
-          0.00024167407536879182,
-          0.00028066313825547695,
-          -0.00041629592305980623,
-          -0.0004892397555522621,
-          -0.0026209561619907618,
-          -0.0029356726445257664,
-          0.000425610167440027,
-          0.0005418087239377201,
-          0.0002375440817559138,
-          -7.489288691431284e-05,
-          -0.0006585267838090658,
-          0.00040605987305752933,
-          -0.0009330196771770716,
-          0.0008937990060076118,
-          -0.0009785185102373362,
-          -0.0005338399787433445,
-          -0.0027988061774522066,
-          -0.004214102402329445,
-          0.0025785816833376884,
-          0.002450671512633562,
-          0.0005351606523618102,
-          0.0012349870521575212,
-          0.0009405002347193658,
-          -0.0011168736964464188,
-          -0.0011542305583134294,
-          -0.0015697323251515627,
-          -0.0005699749453924596,
-          0.0014514662325382233,
-          0.0024399266112595797,
-          0.0031583067029714584,
-          0.0009236747864633799,
-          -0.00357811083085835,
-          -0.0010650111362338066,
-          -0.000355880125425756,
-          -0.0005624890327453613,
-          -1.1902564438059926e-05,
-          0.0011530898045748472,
-          0.0007360299350693822,
-          0.0016493811272084713,
-          0.0008800593786872923,
-          -0.0006905524642206728,
-          -0.0030319970101118088,
-          0.0008080819388851523,
-          0.00010364333866164088,
-          -0.0005807871930301189,
-          -0.0011067159939557314,
-          -0.0026376438327133656,
-          0.00026929602609016,
-          -0.001641781534999609,
-          -0.0034070422407239676,
-          0.001744971377775073,
-          0.00046448659850284457,
-          -0.000789930927567184,
-          0.0018328832229599357,
-          -0.0008632286335341632,
-          -0.00039789103902876377,
-          0.000787912926170975,
-          -0.00012039250577799976,
-          0.0008688485249876976,
-          0.0009473453392274678,
-          -0.0022809950169175863,
-          -0.0011803111992776394,
-          0.00024082366144284606,
-          -0.0004318496794439852,
-          -0.0003728233277797699,
-          -0.0007385354256257415,
-          0.0008114463416859508,
-          -0.00040441250894218683,
-          -0.007074240129441023,
-          0.003946419805288315,
-          -0.01491759717464447,
-          -0.002280194777995348,
-          0.0022679578978568316,
-          -8.297746535390615e-05,
-          -0.004981024190783501,
-          0.0027670864947140217,
-          0.00626616645604372,
-          -0.0034861797466874123,
-          -0.0013347704662010074,
-          -0.0017918730154633522,
-          -0.001223177881911397,
-          0.000405364902690053,
-          -0.00024617952294647694,
-          -0.0057901544496417046,
-          -0.0004975934862159193,
-          0.14218278229236603,
-          -0.001496216980740428,
-          -0.01900637149810791,
-          0.0031333654187619686,
-          -0.0018582025077193975,
+          -0.002156495349481702,
+          -0.00046507216757163405,
+          0.00024167195078916848,
+          0.0002806607517413795,
+          -0.0004162961558904499,
+          -0.0004892406286671758,
+          -0.0026209622155874968,
+          -0.0029356717132031918,
+          0.0004256087704561651,
+          0.0005418135551735759,
+          0.00023754549329169095,
+          -7.4885378126055e-05,
+          -0.0006585237570106983,
+          0.00040605897083878517,
+          -0.0009330313769169152,
+          0.0008937875390984118,
+          -0.0009785089641809464,
+          -0.0005338447517715394,
+          -0.002798808738589287,
+          -0.0042141154408454895,
+          0.0025785760954022408,
+          0.002450673608109355,
+          0.0005351657164283097,
+          0.0012349945027381182,
+          0.0009405009332112968,
+          -0.0011168718338012695,
+          -0.0011542299762368202,
+          -0.001569736865349114,
+          -0.0005699718603864312,
+          0.0014514633221551776,
+          0.0024399259127676487,
+          0.003158304840326309,
+          0.0009236858459189534,
+          -0.0035781036131083965,
+          -0.001065023592673242,
+          -0.00035587642923928797,
+          -0.0005624899640679359,
+          -1.1904543498530984e-05,
+          0.0011530885240063071,
+          0.0007360292365774512,
+          0.0016493821749463677,
+          0.0008800605428405106,
+          -0.0006905477494001389,
+          -0.0030319932848215103,
+          0.0008080820553004742,
+          0.00010364956688135862,
+          -0.000580791849642992,
+          -0.0011067147133871913,
+          -0.0026376410387456417,
+          0.0002693022252060473,
+          -0.0016417787410318851,
+          -0.0034070289693772793,
+          0.0017449733568355441,
+          0.00046448662760667503,
+          -0.0007899348856881261,
+          0.0018328834557905793,
+          -0.0008632306708022952,
+          -0.00039788917638361454,
+          0.0007879154873080552,
+          -0.00012039285502396524,
+          0.0008688486414030194,
+          0.0009473506361246109,
+          -0.002280982444062829,
+          -0.0011803142260760069,
+          0.0002408194704912603,
+          -0.0004318461287766695,
+          -0.00037281878758221865,
+          -0.0007385292556136847,
+          0.0008114340016618371,
+          -0.00040442857425659895,
+          -0.007074241526424885,
+          0.003946421667933464,
+          -0.014917591586709023,
+          -0.0022801952436566353,
+          0.0022679539397358894,
+          -8.297240128740668e-05,
+          -0.0049810330383479595,
+          0.0027670827694237232,
+          0.0062661729753017426,
+          -0.0034861767198890448,
+          -0.0013347702333703637,
+          -0.001791873830370605,
+          -0.0012231743894517422,
+          0.0004053567536175251,
+          -0.0002461824333295226,
+          -0.005790156312286854,
+          -0.0004975919146090746,
+          0.14218276739120483,
+          -0.0014962118584662676,
+          -0.01900637522339821,
+          0.003133367281407118,
+          -0.0018581977346912026,
           -0.011305071413516998,
-          0.19225990772247314,
-          -0.0011892484035342932,
-          -0.0010282367002218962,
-          -0.0038004019297659397,
-          -0.0008571128128096461,
-          -0.013956686481833458,
-          0.008289236575365067,
-          0.004314992111176252,
-          -0.009073692373931408,
-          -0.08315175771713257,
-          0.0034568109549582005,
-          -0.018055014312267303,
-          0.00217800703831017,
+          0.19225995242595673,
+          -0.0011892454931512475,
+          -0.0010282358853146434,
+          -0.0038004028610885143,
+          -0.0008571104845032096,
+          -0.013956702314317226,
+          0.008289228193461895,
+          0.004315000958740711,
+          -0.009073707275092602,
+          -0.08315176516771317,
+          0.003456807229667902,
+          -0.01805501990020275,
+          0.00217801658436656,
           0.29780468344688416,
-          0.02409377135336399,
+          0.024093760177493095,
           0.08904657512903214,
-          -0.0007931506261229515,
-          0.07247606664896011,
-          0.015016509220004082,
-          -0.021209245547652245,
-          0.052054572850465775,
+          -0.0007931562722660601,
+          0.07247605919837952,
+          0.01501650083810091,
+          -0.021209243685007095,
+          0.05205458402633667,
           1.441115140914917,
           0.04743725806474686,
           -0.0322909839451313,
           0.0,
-          0.0019993034657090902,
-          -0.008072325959801674,
-          0.8600812554359436,
-          0.3260071277618408,
+          0.0019993027672171593,
+          -0.008072329685091972,
+          0.8600811958312988,
+          0.32600724697113037,
           0.16344299912452698,
-          0.07133564352989197,
+          0.07133562117815018,
           -0.004448350518941879,
-          0.0006813818472437561,
-          0.36613476276397705,
-          -0.710505485534668,
-          -0.0020313779823482037,
-          -0.032143473625183105,
-          1.2294319868087769,
-          0.0018451482756063342,
-          0.016877301037311554,
-          -0.001730009913444519,
-          -0.501085638999939,
-          0.02749774605035782,
-          -0.005966278724372387,
-          -0.0049441163428127766,
+          0.0006813746877014637,
+          0.3661348223686218,
+          -0.7105053663253784,
+          -0.0020313761197030544,
+          -0.03214346989989281,
+          1.2294316291809082,
+          0.0018451516516506672,
+          0.016877306625247,
+          -0.0017300043255090714,
+          -0.5010855793952942,
+          0.02749773859977722,
+          -0.005966317839920521,
+          -0.004944117274135351,
           -0.08855222165584564,
-          0.006622320972383022,
-          0.04412432014942169,
-          -0.027266837656497955,
+          0.006622340530157089,
+          0.04412432760000229,
+          -0.02726682648062706,
           -1.1349124908447266,
-          0.02287965640425682
+          0.022879652678966522
          ],
          "xaxis": "x",
          "y": [
-          -0.0020563285797834396,
-          -0.0005101955030113459,
-          0.0004685768508352339,
-          0.00012511832755990326,
-          -0.0006028721109032631,
-          -0.00024295502225868404,
-          -0.002318894723430276,
-          -0.002758359769359231,
-          0.0005645868368446827,
-          0.000969740329310298,
-          -0.0002504501899238676,
-          4.740082658827305e-06,
-          -0.0010071131400763988,
-          0.0003947088844142854,
-          -0.0015487205237150192,
-          0.0014034901978448033,
-          -0.0012652688892558217,
-          -0.0011358250631019473,
-          -0.0028159404173493385,
-          -0.0029645359609276056,
+          -0.002056329045444727,
+          -0.0005102002760395408,
+          0.0004685759777203202,
+          0.00012511858949437737,
+          -0.0006028746138326824,
+          -0.00024295759794767946,
+          -0.0023188991472125053,
+          -0.0027583539485931396,
+          0.000564592657610774,
+          0.0009697366622276604,
+          -0.0002504411095287651,
+          4.7396752052009106e-06,
+          -0.0010071106953546405,
+          0.0003947066143155098,
+          -0.001548723317682743,
+          0.0014034844934940338,
+          -0.0012652697041630745,
+          -0.0011358254123479128,
+          -0.0028159399516880512,
+          -0.0029645331669598818,
           0.0029190238565206528,
           0.0025743518490344286,
           0.00036237656604498625,
-          0.0017548884497955441,
-          0.0005569332861341536,
-          -0.00112663593608886,
-          -0.0017354178708046675,
-          -0.0014514722861349583,
-          -0.00028740704874508083,
-          0.0017210595542564988,
-          0.00266590085811913,
-          0.0031146793626248837,
-          0.0005667305667884648,
-          -0.0036664949730038643,
-          -0.0018847067840397358,
-          7.027178071439266e-06,
-          -0.000726439815480262,
-          0.00011369686399120837,
-          0.001430142787285149,
-          0.000749052269384265,
-          0.0020184761378914118,
-          0.0007436758605763316,
-          -0.0004617759259417653,
-          -0.003905785735696554,
-          0.001140733016654849,
-          -4.027335671707988e-05,
-          -0.0013293256051838398,
-          -0.0017636881675571203,
-          -0.002828173339366913,
-          0.0003364472358953208,
-          -0.0014249038649722934,
-          -0.0037773081567138433,
-          0.0015998876187950373,
-          0.00029889732832089067,
-          -0.0008046309230849147,
-          0.0020388164557516575,
-          -0.0015593776479363441,
-          -0.0006437147385440767,
-          0.001116806990467012,
-          -0.00035003889934159815,
-          0.0011338151525706053,
-          0.0011259106686338782,
-          -0.0025163597892969847,
-          -0.0014790240675210953,
-          0.00038791983388364315,
-          -6.407807813957334e-05,
-          -0.0005096746608614922,
-          -0.0008841876406222582,
-          0.0006399309495463967,
-          -0.001009696745313704,
-          -0.0067590330727398396,
-          0.0033667273819446564,
-          -0.015147387981414795,
-          -0.002135086804628372,
-          0.002593189012259245,
-          -0.00042674108408391476,
-          -0.005559002980589867,
-          0.0026659294962882996,
-          0.006410874892026186,
-          -0.0038270100485533476,
-          -0.00038422830402851105,
-          -0.0016430210089311004,
-          -0.0013344308827072382,
-          -9.184109512716532e-05,
-          -9.488123760093004e-05,
-          -0.005788922309875488,
-          -0.0006383719155564904,
+          0.0017548904288560152,
+          0.0005569161148741841,
+          -0.0011266364017501473,
+          -0.0017354193842038512,
+          -0.0014514740323647857,
+          -0.0002874041674658656,
+          0.0017210585065186024,
+          0.0026659027207642794,
+          0.0031146786641329527,
+          0.0005667298682965338,
+          -0.003666497301310301,
+          -0.0018847138853743672,
+          7.028633262962103e-06,
+          -0.0007264401647262275,
+          0.0001136981591116637,
+          0.0014301439514383674,
+          0.0007490518037229776,
+          0.002018478699028492,
+          0.0007436765008606017,
+          -0.0004617785452865064,
+          -0.0039057875983417034,
+          0.0011407355777919292,
+          -4.027353134006262e-05,
+          -0.0013293151278048754,
+          -0.0017636835109442472,
+          -0.0028281747363507748,
+          0.0003364539588801563,
+          -0.0014249025844037533,
+          -0.003777292789891362,
+          0.001599886454641819,
+          0.00029889593133702874,
+          -0.0008046383736655116,
+          0.0020388178527355194,
+          -0.001559373107738793,
+          -0.0006437154370360076,
+          0.001116809668019414,
+          -0.0003500327584333718,
+          0.0011338141048327088,
+          0.0011259093880653381,
+          -0.0025163700338453054,
+          -0.0014790259301662445,
+          0.00038791849510744214,
+          -6.407558976206928e-05,
+          -0.0005096771637909114,
+          -0.0008841846138238907,
+          0.0006399258272722363,
+          -0.0010097046615555882,
+          -0.006759032607078552,
+          0.0033667325042188168,
+          -0.015147397294640541,
+          -0.0021350914612412453,
+          0.0025931934360414743,
+          -0.00042673421557992697,
+          -0.005559004843235016,
+          0.002665933221578598,
+          0.0064108846709132195,
+          -0.0038270088844001293,
+          -0.0003842375008389354,
+          -0.0016430213581770658,
+          -0.001334429020062089,
+          -9.183748625218868e-05,
+          -9.488424984738231e-05,
+          -0.005788922775536776,
+          -0.0006383699947036803,
           0.134933739900589,
-          -0.0017687628278508782,
+          -0.0017687629442662,
           -0.018917974084615707,
-          0.0038733629044145346,
-          -0.002145076170563698,
-          -0.010327237658202648,
-          0.18325874209403992,
-          -0.0007747883792035282,
-          -0.0010452116839587688,
-          -0.003833947703242302,
-          -0.000804627372417599,
-          -0.012673338875174522,
-          0.008045812137424946,
-          0.0036040153354406357,
-          -0.009398169815540314,
+          0.0038733729161322117,
+          -0.0021450738422572613,
+          -0.010327240452170372,
+          0.1832587718963623,
+          -0.0007747872150503099,
+          -0.0010452070273458958,
+          -0.0038339472375810146,
+          -0.0008046274306252599,
+          -0.01267334446310997,
+          0.008045826107263565,
+          0.003604009747505188,
+          -0.009398158639669418,
           -0.08272106945514679,
-          0.0035550352185964584,
-          -0.018404126167297363,
-          0.0017586719477549195,
-          0.2896132469177246,
-          0.022854045033454895,
-          0.08595201373100281,
-          -0.0006932567339390516,
-          0.06816966831684113,
-          0.01311141811311245,
-          -0.021098004654049873,
-          0.05112440511584282,
+          0.0035550370812416077,
+          -0.01840413361787796,
+          0.001758674974553287,
+          0.28961312770843506,
+          0.022854033857584,
+          0.08595199882984161,
+          -0.000693259877152741,
+          0.06816964596509933,
+          0.01311142835766077,
+          -0.02109798789024353,
+          0.05112443491816521,
           1.384489893913269,
-          0.04583733528852463,
-          -0.038303446024656296,
-          2.9854445457458496,
-          0.001966139767318964,
-          -0.008030213415622711,
-          0.560872495174408,
+          0.045837316662073135,
+          -0.038303449749946594,
+          2.9854443073272705,
+          0.001966138370335102,
+          -0.008030208759009838,
+          0.5608724355697632,
           0.17083144187927246,
-          -0.033618733286857605,
-          0.05821547657251358,
-          -0.0024530075024813414,
-          0.0018771879840642214,
-          0.2882729768753052,
-          -1.898641586303711,
-          -0.001528693363070488,
-          -0.03513003885746002,
+          -0.033618729561567307,
+          0.05821548402309418,
+          -0.0024530175141990185,
+          0.0018771894974634051,
+          0.2882729470729828,
+          -1.8986413478851318,
+          -0.0015286938287317753,
+          -0.035130057483911514,
           0.48021769523620605,
-          -0.0009116916917264462,
-          0.0160758625715971,
-          -0.03986112400889397,
+          -0.0009116912842728198,
+          0.016075868159532547,
+          -0.03986111283302307,
           -0.3879111707210541,
-          0.011123226955533028,
-          -0.005477802362293005,
-          -0.0025129495188593864,
-          -0.08056114614009857,
-          0.007518642581999302,
-          0.043011054396629333,
-          -0.0400824099779129,
-          -0.9702335596084595,
-          0.011862391605973244
+          0.01112320739775896,
+          -0.0054778107441961765,
+          -0.002512941136956215,
+          -0.08056112378835678,
+          0.007518645375967026,
+          0.04301108419895172,
+          -0.040082402527332306,
+          -0.9702335000038147,
+          0.011862380430102348
          ],
          "yaxis": "y"
         }
@@ -21018,9 +21018,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"38925f09-980d-4e2f-8ebb-727438003d35\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"38925f09-980d-4e2f-8ebb-727438003d35\")) {                    Plotly.newPlot(                        \"38925f09-980d-4e2f-8ebb-727438003d35\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],\"xaxis\":\"x\",\"y\":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"69a381a6-98d5-41d7-b432-3b083c9cf66c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"69a381a6-98d5-41d7-b432-3b083c9cf66c\")) {                    Plotly.newPlot(                        \"69a381a6-98d5-41d7-b432-3b083c9cf66c\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156495349481702,-0.00046507216757163405,0.00024167195078916848,0.0002806607517413795,-0.0004162961558904499,-0.0004892406286671758,-0.0026209622155874968,-0.0029356717132031918,0.0004256087704561651,0.0005418135551735759,0.00023754549329169095,-7.4885378126055e-05,-0.0006585237570106983,0.00040605897083878517,-0.0009330313769169152,0.0008937875390984118,-0.0009785089641809464,-0.0005338447517715394,-0.002798808738589287,-0.0042141154408454895,0.0025785760954022408,0.002450673608109355,0.0005351657164283097,0.0012349945027381182,0.0009405009332112968,-0.0011168718338012695,-0.0011542299762368202,-0.001569736865349114,-0.0005699718603864312,0.0014514633221551776,0.0024399259127676487,0.003158304840326309,0.0009236858459189534,-0.0035781036131083965,-0.001065023592673242,-0.00035587642923928797,-0.0005624899640679359,-1.1904543498530984e-05,0.0011530885240063071,0.0007360292365774512,0.0016493821749463677,0.0008800605428405106,-0.0006905477494001389,-0.0030319932848215103,0.0008080820553004742,0.00010364956688135862,-0.000580791849642992,-0.0011067147133871913,-0.0026376410387456417,0.0002693022252060473,-0.0016417787410318851,-0.0034070289693772793,0.0017449733568355441,0.00046448662760667503,-0.0007899348856881261,0.0018328834557905793,-0.0008632306708022952,-0.00039788917638361454,0.0007879154873080552,-0.00012039285502396524,0.0008688486414030194,0.0009473506361246109,-0.002280982444062829,-0.0011803142260760069,0.0002408194704912603,-0.0004318461287766695,-0.00037281878758221865,-0.0007385292556136847,0.0008114340016618371,-0.00040442857425659895,-0.007074241526424885,0.003946421667933464,-0.014917591586709023,-0.0022801952436566353,0.0022679539397358894,-8.297240128740668e-05,-0.0049810330383479595,0.0027670827694237232,0.0062661729753017426,-0.0034861767198890448,-0.0013347702333703637,-0.001791873830370605,-0.0012231743894517422,0.0004053567536175251,-0.0002461824333295226,-0.005790156312286854,-0.0004975919146090746,0.14218276739120483,-0.0014962118584662676,-0.01900637522339821,0.003133367281407118,-0.0018581977346912026,-0.011305071413516998,0.19225995242595673,-0.0011892454931512475,-0.0010282358853146434,-0.0038004028610885143,-0.0008571104845032096,-0.013956702314317226,0.008289228193461895,0.004315000958740711,-0.009073707275092602,-0.08315176516771317,0.003456807229667902,-0.01805501990020275,0.00217801658436656,0.29780468344688416,0.024093760177493095,0.08904657512903214,-0.0007931562722660601,0.07247605919837952,0.01501650083810091,-0.021209243685007095,0.05205458402633667,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993027672171593,-0.008072329685091972,0.8600811958312988,0.32600724697113037,0.16344299912452698,0.07133562117815018,-0.004448350518941879,0.0006813746877014637,0.3661348223686218,-0.7105053663253784,-0.0020313761197030544,-0.03214346989989281,1.2294316291809082,0.0018451516516506672,0.016877306625247,-0.0017300043255090714,-0.5010855793952942,0.02749773859977722,-0.005966317839920521,-0.004944117274135351,-0.08855222165584564,0.006622340530157089,0.04412432760000229,-0.02726682648062706,-1.1349124908447266,0.022879652678966522],\"xaxis\":\"x\",\"y\":[-0.002056329045444727,-0.0005102002760395408,0.0004685759777203202,0.00012511858949437737,-0.0006028746138326824,-0.00024295759794767946,-0.0023188991472125053,-0.0027583539485931396,0.000564592657610774,0.0009697366622276604,-0.0002504411095287651,4.7396752052009106e-06,-0.0010071106953546405,0.0003947066143155098,-0.001548723317682743,0.0014034844934940338,-0.0012652697041630745,-0.0011358254123479128,-0.0028159399516880512,-0.0029645331669598818,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548904288560152,0.0005569161148741841,-0.0011266364017501473,-0.0017354193842038512,-0.0014514740323647857,-0.0002874041674658656,0.0017210585065186024,0.0026659027207642794,0.0031146786641329527,0.0005667298682965338,-0.003666497301310301,-0.0018847138853743672,7.028633262962103e-06,-0.0007264401647262275,0.0001136981591116637,0.0014301439514383674,0.0007490518037229776,0.002018478699028492,0.0007436765008606017,-0.0004617785452865064,-0.0039057875983417034,0.0011407355777919292,-4.027353134006262e-05,-0.0013293151278048754,-0.0017636835109442472,-0.0028281747363507748,0.0003364539588801563,-0.0014249025844037533,-0.003777292789891362,0.001599886454641819,0.00029889593133702874,-0.0008046383736655116,0.0020388178527355194,-0.001559373107738793,-0.0006437154370360076,0.001116809668019414,-0.0003500327584333718,0.0011338141048327088,0.0011259093880653381,-0.0025163700338453054,-0.0014790259301662445,0.00038791849510744214,-6.407558976206928e-05,-0.0005096771637909114,-0.0008841846138238907,0.0006399258272722363,-0.0010097046615555882,-0.006759032607078552,0.0033667325042188168,-0.015147397294640541,-0.0021350914612412453,0.0025931934360414743,-0.00042673421557992697,-0.005559004843235016,0.002665933221578598,0.0064108846709132195,-0.0038270088844001293,-0.0003842375008389354,-0.0016430213581770658,-0.001334429020062089,-9.183748625218868e-05,-9.488424984738231e-05,-0.005788922775536776,-0.0006383699947036803,0.134933739900589,-0.0017687629442662,-0.018917974084615707,0.0038733729161322117,-0.0021450738422572613,-0.010327240452170372,0.1832587718963623,-0.0007747872150503099,-0.0010452070273458958,-0.0038339472375810146,-0.0008046274306252599,-0.01267334446310997,0.008045826107263565,0.003604009747505188,-0.009398158639669418,-0.08272106945514679,0.0035550370812416077,-0.01840413361787796,0.001758674974553287,0.28961312770843506,0.022854033857584,0.08595199882984161,-0.000693259877152741,0.06816964596509933,0.01311142835766077,-0.02109798789024353,0.05112443491816521,1.384489893913269,0.045837316662073135,-0.038303449749946594,2.9854443073272705,0.001966138370335102,-0.008030208759009838,0.5608724355697632,0.17083144187927246,-0.033618729561567307,0.05821548402309418,-0.0024530175141990185,0.0018771894974634051,0.2882729470729828,-1.8986413478851318,-0.0015286938287317753,-0.035130057483911514,0.48021769523620605,-0.0009116912842728198,0.016075868159532547,-0.03986111283302307,-0.3879111707210541,0.01112320739775896,-0.0054778107441961765,-0.002512941136956215,-0.08056112378835678,0.007518645375967026,0.04301108419895172,-0.040082402527332306,-0.9702335000038147,0.011862380430102348],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('38925f09-980d-4e2f-8ebb-727438003d35');\n",
+       "var gd = document.getElementById('69a381a6-98d5-41d7-b432-3b083c9cf66c');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -21085,10 +21085,10 @@
    "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:24.910983Z",
-     "iopub.status.busy": "2024-11-19T14:50:24.910634Z",
-     "iopub.status.idle": "2024-11-19T14:50:24.915819Z",
-     "shell.execute_reply": "2024-11-19T14:50:24.915375Z"
+     "iopub.execute_input": "2024-12-14T01:01:15.447724Z",
+     "iopub.status.busy": "2024-12-14T01:01:15.447522Z",
+     "iopub.status.idle": "2024-12-14T01:01:15.452855Z",
+     "shell.execute_reply": "2024-12-14T01:01:15.452388Z"
     }
    },
    "outputs": [
@@ -21164,7 +21164,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.10"
+   "version": "3.11.11"
   },
   "vscode": {
    "interpreter": {
@@ -21174,106 +21174,47 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "0453b85ea3f4445e9c9a42ebae748343": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "0542760fcf0c421e913bc2b839ac792c": {
+     "0947963163124b84ad7bdbd0d14a5852": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_8446cb0777b7432c9f6f15127fae1ec1",
-       "placeholder": "​",
-       "style": "IPY_MODEL_25355715e4f84b3fb30b7fb7d73dab5b",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 548M/548M [00:02&lt;00:00, 245MB/s]"
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "080b9c4c47814b0a82c5dea8974cab19": {
+     "0b727299d6b94848ba14d79205e0d847": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_877a40ace6ae4c65af3c6d4b089520da",
-       "placeholder": "​",
-       "style": "IPY_MODEL_6147070de83046bca741d4491c2fbea6",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_d3bf56862e814e46a08ce4949ce5df67",
+        "IPY_MODEL_9a499b42e58849dcbfc030ebb0630a62",
+        "IPY_MODEL_b4faa038d86840faa63b4ff7f8ac404c"
+       ],
+       "layout": "IPY_MODEL_5fe7263a54b34f2b918ca2c57d194d74",
        "tabbable": null,
-       "tooltip": null,
-       "value": "merges.txt: 100%"
+       "tooltip": null
       }
      },
-     "10327cf5d6bb477c9888e49d40d9b6f4": {
+     "0ccd5c3ba66d4687bda0b8a38511eecd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -21288,88 +21229,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_c14c9494085f4f6bb025489a0d6ccaed",
+       "layout": "IPY_MODEL_b60c8dc4072a47f69bc60e5115ab87c6",
        "placeholder": "​",
-       "style": "IPY_MODEL_88d1d898152b478c94604af43ac42d12",
+       "style": "IPY_MODEL_3dd3706dad6149a2ae71bfb8a4a5256a",
        "tabbable": null,
        "tooltip": null,
        "value": "model.safetensors: 100%"
       }
      },
-     "152fb19b656b444982eb11fae36392ae": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_8adc10691f3b407aa401a7cee384d572",
-       "placeholder": "​",
-       "style": "IPY_MODEL_4dadf0b8acff4ab9a4d61fc31ce775ec",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer_config.json: 100%"
-      }
-     },
-     "1c7bc6d96a044cb69b7e5e3dc190b2f3": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_44931b936f7e41a88aa39b6df566c649",
-       "max": 548105171.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_68329682dcdf4df08289a498e7f1ca58",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 548105171.0
-      }
-     },
-     "1fd27b66709b400bae1bba1dd1353b37": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_24477b292cec43c38a49e1e7782f5827",
-        "IPY_MODEL_949ab7013dd74901865d601116408100",
-        "IPY_MODEL_c1293200aaf4409a8b484e48a74056bb"
-       ],
-       "layout": "IPY_MODEL_e79d7678437a45a39ca514276be1e4a9",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "24477b292cec43c38a49e1e7782f5827": {
+     "0dc2fdfccac3440ea6a358b4b727da08": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -21384,33 +21252,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_2fc1d8a4705f4cf180f6066cdb6000d7",
+       "layout": "IPY_MODEL_910235bb167a412ab2a54c930635b3bf",
        "placeholder": "​",
-       "style": "IPY_MODEL_7dbd77370d9a4010a60276a6071b73bd",
+       "style": "IPY_MODEL_f988b1609264444ea2e348b491a028a4",
        "tabbable": null,
        "tooltip": null,
-       "value": "tokenizer.json: 100%"
-      }
-     },
-     "25355715e4f84b3fb30b7fb7d73dab5b": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 47.4MB/s]"
       }
      },
-     "2fc1d8a4705f4cf180f6066cdb6000d7": {
+     "1acc38b256394079bf479fc9c3f6bbff": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21463,7 +21313,7 @@
        "width": null
       }
      },
-     "3261acc0fd974a1686a4fc899a6ba62f": {
+     "1dffcf97acf041dfa4829930fc637e56": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21516,25 +21366,31 @@
        "width": null
       }
      },
-     "32671fce2b1942d8b7d158d4706c2713": {
+     "2566b58370ab4ee1a23ec40b298e524d": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_0ccd5c3ba66d4687bda0b8a38511eecd",
+        "IPY_MODEL_2a992a646fbc405db6ec21ba17538e0e",
+        "IPY_MODEL_ddeeebc813c54775ab8841a88e512534"
+       ],
+       "layout": "IPY_MODEL_1acc38b256394079bf479fc9c3f6bbff",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "3328cd9e412b4e8ea246fbb324d064c2": {
+     "2a992a646fbc405db6ec21ba17538e0e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -21550,35 +21406,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_4e0363da178140e38a1f5d5a763d8b9d",
-       "max": 124.0,
+       "layout": "IPY_MODEL_fe14fba431ee44719e467a94d38602c1",
+       "max": 548105171.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_b071093461004612aea25c3faaa56d75",
+       "style": "IPY_MODEL_71480013d795482ea7827866fd7d1fb0",
        "tabbable": null,
        "tooltip": null,
-       "value": 124.0
-      }
-     },
-     "34dc2dd7d1b644148cf1162dd8fffae2": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "value": 548105171.0
       }
      },
-     "36392e12f266416ea89267464302a9ae": {
+     "3250ac219bb14a0d87380c723d7187c5": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21631,47 +21469,25 @@
        "width": null
       }
      },
-     "372cd65f585d4674944843e298605bc0": {
+     "3dd3706dad6149a2ae71bfb8a4a5256a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "3790ba28df6f45f098a68fcec3f968f8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_080b9c4c47814b0a82c5dea8974cab19",
-        "IPY_MODEL_a293e3738c334e4c9cc1bb9c71a81dc7",
-        "IPY_MODEL_8e014224685042fb868f27a39ba7175f"
-       ],
-       "layout": "IPY_MODEL_5b1acd258811455882cfb880a8d4790e",
-       "tabbable": null,
-       "tooltip": null
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "39b9f02d659f4e00ae2652e1cd790372": {
+     "4064ed6431a14c8d940329269c2399e3": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21689,7 +21505,7 @@
        "text_color": null
       }
      },
-     "3b61ee1aea87467da549d7a70b026ce9": {
+     "4647f9260ab248d7a350e7a07c999a9b": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21742,7 +21558,23 @@
        "width": null
       }
      },
-     "44931b936f7e41a88aa39b6df566c649": {
+     "47513102e87a47069219c229c4725eea": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "481fa4fd501940cebdf3f44be6c9e882": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21795,7 +21627,7 @@
        "width": null
       }
      },
-     "449a255401944642930f3f2821a91d6f": {
+     "488e5f543a8f48a79b17dd90afc28284": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21848,7 +21680,7 @@
        "width": null
       }
      },
-     "4ab50487fcca41b59f1392f90cdc6757": {
+     "4973dab052f14376aa999fd6bc13197b": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21901,58 +21733,14 @@
        "width": null
       }
      },
-     "4ad663a579114d4eb47fd1364f66ccc0": {
-      "model_module": "@jupyter-widgets/controls",
+     "4a23001ae4824747ac51dbe487a0a126": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_cb2bcca49ce84b5f813a3e09a28e17e8",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_7de92e4bb8af4843b82528edf005b6c3",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 26.0
-      }
-     },
-     "4dadf0b8acff4ab9a4d61fc31ce775ec": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "4dc07cf1dbcb4f518a591b3bf3de91df": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
@@ -21998,7 +21786,31 @@
        "width": null
       }
      },
-     "4e0363da178140e38a1f5d5a763d8b9d": {
+     "4c4a1ec2b7514c1297506faae67c1374": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_53bd3bcec1c747a0a2f91fd51bd8fba0",
+        "IPY_MODEL_53f61b927c1346d88f4d1f2962f875ba",
+        "IPY_MODEL_c43b6d1b7dc04e8ea921c5940f890126"
+       ],
+       "layout": "IPY_MODEL_b1a0d471042a4a018d4de3bf29a3ecd2",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "4faadec4ac0f4a4ab32961fb3b74e529": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22051,7 +21863,7 @@
        "width": null
       }
      },
-     "5604e63f816e45f280635edb76818789": {
+     "53bd3bcec1c747a0a2f91fd51bd8fba0": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22066,15 +21878,65 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_3b61ee1aea87467da549d7a70b026ce9",
+       "layout": "IPY_MODEL_481fa4fd501940cebdf3f44be6c9e882",
        "placeholder": "​",
-       "style": "IPY_MODEL_5f44a32ffc6d4553ae2b471e7843b288",
+       "style": "IPY_MODEL_ed04372d5ce8433d848d96e975723537",
        "tabbable": null,
        "tooltip": null,
-       "value": "vocab.json: 100%"
+       "value": "generation_config.json: 100%"
+      }
+     },
+     "53f61b927c1346d88f4d1f2962f875ba": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f06d8ed2f2fc40c78e6f71a980042fa2",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_0947963163124b84ad7bdbd0d14a5852",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 124.0
+      }
+     },
+     "54e358754fd241a7beb89dbd18d54469": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_7daabc68020641e897d4809176829307",
+        "IPY_MODEL_ba3727d9eb8e45b394738da6078d874c",
+        "IPY_MODEL_ac60beb4421b4e758aec659715fa5146"
+       ],
+       "layout": "IPY_MODEL_97133900a1a94bc691ce291364fca64c",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "5b1acd258811455882cfb880a8d4790e": {
+     "5fe7263a54b34f2b918ca2c57d194d74": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22127,66 +21989,102 @@
        "width": null
       }
      },
-     "5ce85280f03e416599efda8942bfd6b7": {
-      "model_module": "@jupyter-widgets/controls",
+     "621583ed869b478d913fc04a87073a95": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_36392e12f266416ea89267464302a9ae",
-       "placeholder": "​",
-       "style": "IPY_MODEL_39b9f02d659f4e00ae2652e1cd790372",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 21.0kB/s]"
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "5f44a32ffc6d4553ae2b471e7843b288": {
+     "6ab9eaf5ee3342379119468f880a7e68": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "ProgressStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "6147070de83046bca741d4491c2fbea6": {
+     "6b5428d552484be2b498aa06f2cd12d7": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "FloatProgressModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_488e5f543a8f48a79b17dd90afc28284",
+       "max": 1042301.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_7ffcda766f894799bb645944b5ecab6b",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1042301.0
       }
      },
-     "68329682dcdf4df08289a498e7f1ca58": {
+     "71480013d795482ea7827866fd7d1fb0": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "ProgressStyleModel",
@@ -22202,7 +22100,30 @@
        "description_width": ""
       }
      },
-     "68b77f75328e4a539b09b13b20cac43b": {
+     "7daabc68020641e897d4809176829307": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_3250ac219bb14a0d87380c723d7187c5",
+       "placeholder": "​",
+       "style": "IPY_MODEL_f6e7f731edb44410a1bbe854b20b75b3",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer_config.json: 100%"
+      }
+     },
+     "7f602aa1ed364df1a417f0cf429e4a5c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22220,7 +22141,23 @@
        "text_color": null
       }
      },
-     "729153cf51e244f598470b0f440f9b25": {
+     "7ffcda766f894799bb645944b5ecab6b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "853700c97ecf475a8336c57945c87c6a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22273,25 +22210,7 @@
        "width": null
       }
      },
-     "77914f1f07d648eab888c503c2a32cf0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "77db344030f64e1cb37bde70ed8e7676": {
+     "860eac54afd644cdb66576f10d1c4b94": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22344,48 +22263,14 @@
        "width": null
       }
      },
-     "7dbd77370d9a4010a60276a6071b73bd": {
-      "model_module": "@jupyter-widgets/controls",
+     "910235bb167a412ab2a54c930635b3bf": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "7de92e4bb8af4843b82528edf005b6c3": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "7f9b1c85b4da4b068ea5593efc94d962": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
@@ -22431,7 +22316,25 @@
        "width": null
       }
      },
-     "822ddcdd61b443ce9bc565addafb6984": {
+     "924d96b2376c4b16a4541d64b0e45872": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "9351dee6b05946a0a6880580b6f6ff6b": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22484,7 +22387,7 @@
        "width": null
       }
      },
-     "8446cb0777b7432c9f6f15127fae1ec1": {
+     "97133900a1a94bc691ce291364fca64c": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22537,7 +22440,97 @@
        "width": null
       }
      },
-     "877a40ace6ae4c65af3c6d4b089520da": {
+     "9a499b42e58849dcbfc030ebb0630a62": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_4faadec4ac0f4a4ab32961fb3b74e529",
+       "max": 456318.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_6ab9eaf5ee3342379119468f880a7e68",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 456318.0
+      }
+     },
+     "aa33f2b3f7af4a5c956ebdbae08dabec": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_4973dab052f14376aa999fd6bc13197b",
+       "placeholder": "​",
+       "style": "IPY_MODEL_4064ed6431a14c8d940329269c2399e3",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer.json: 100%"
+      }
+     },
+     "ac60beb4421b4e758aec659715fa5146": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_c0e90bf5be684b4eadbd6f97b968ab31",
+       "placeholder": "​",
+       "style": "IPY_MODEL_e49babdcc96f4f99af8dba7cd03202fc",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 26.0/26.0 [00:00&lt;00:00, 4.60kB/s]"
+      }
+     },
+     "add5c8f3c40642ef8e4cf96579325ec3": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "b1a0d471042a4a018d4de3bf29a3ecd2": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22590,25 +22583,46 @@
        "width": null
       }
      },
-     "88d1d898152b478c94604af43ac42d12": {
+     "b2dd202e54cc4c4bbf18779cef088e63": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "ProgressStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "b4faa038d86840faa63b4ff7f8ac404c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_de2a35b1b3264fd2ac593af14da5b645",
+       "placeholder": "​",
+       "style": "IPY_MODEL_c02c62e902e648ef9e245988ad7917f6",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 456k/456k [00:00&lt;00:00, 38.7MB/s]"
       }
      },
-     "8adc10691f3b407aa401a7cee384d572": {
+     "b60c8dc4072a47f69bc60e5115ab87c6": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22661,7 +22675,7 @@
        "width": null
       }
      },
-     "8e014224685042fb868f27a39ba7175f": {
+     "b9ffe90723d54a178c02c4a71079ef4f": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22676,33 +22690,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_3261acc0fd974a1686a4fc899a6ba62f",
+       "layout": "IPY_MODEL_9351dee6b05946a0a6880580b6f6ff6b",
        "placeholder": "​",
-       "style": "IPY_MODEL_77914f1f07d648eab888c503c2a32cf0",
+       "style": "IPY_MODEL_924d96b2376c4b16a4541d64b0e45872",
        "tabbable": null,
        "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 29.0MB/s]"
-      }
-     },
-     "90da3925110a458aad1cc1c6f8f98fd4": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "value": "vocab.json: 100%"
       }
      },
-     "949ab7013dd74901865d601116408100": {
+     "ba3727d9eb8e45b394738da6078d874c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -22718,155 +22714,106 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_d811ce6fc2d94b5a8b4a8e4bb7b04432",
-       "max": 1355256.0,
+       "layout": "IPY_MODEL_853700c97ecf475a8336c57945c87c6a",
+       "max": 26.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_372cd65f585d4674944843e298605bc0",
+       "style": "IPY_MODEL_47513102e87a47069219c229c4725eea",
        "tabbable": null,
        "tooltip": null,
-       "value": 1355256.0
+       "value": 26.0
       }
      },
-     "9987dd8fb4df47f8bad2480d55f0644e": {
-      "model_module": "@jupyter-widgets/controls",
+     "bec18b05c0a8413485c9f6fb6d8d51dd": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_cafce154ce254e2489b3c3231cd3afc8",
-       "placeholder": "​",
-       "style": "IPY_MODEL_34dc2dd7d1b644148cf1162dd8fffae2",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "generation_config.json: 100%"
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "a293e3738c334e4c9cc1bb9c71a81dc7": {
+     "bf988798a38d454fa0c118280dbeab56": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_449a255401944642930f3f2821a91d6f",
-       "max": 456318.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_d096417d88bf4cc79562f5b69c150050",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 456318.0
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "adf78455ad4b4255b5ee12b111c8d6bf": {
+     "c02c62e902e648ef9e245988ad7917f6": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_10327cf5d6bb477c9888e49d40d9b6f4",
-        "IPY_MODEL_1c7bc6d96a044cb69b7e5e3dc190b2f3",
-        "IPY_MODEL_0542760fcf0c421e913bc2b839ac792c"
-       ],
-       "layout": "IPY_MODEL_0453b85ea3f4445e9c9a42ebae748343",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "b071093461004612aea25c3faaa56d75": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "bba7279dfd574dd5b910973cde0fc709": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4dc07cf1dbcb4f518a591b3bf3de91df",
-       "max": 1042301.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_ccd9adc54d384d26a6e04c69df253074",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1042301.0
-      }
-     },
-     "c1293200aaf4409a8b484e48a74056bb": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_d310211006ce4cb6aac419247e69239f",
-       "placeholder": "​",
-       "style": "IPY_MODEL_32671fce2b1942d8b7d158d4706c2713",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 4.54MB/s]"
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "c14c9494085f4f6bb025489a0d6ccaed": {
+     "c0e90bf5be684b4eadbd6f97b968ab31": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22919,60 +22866,30 @@
        "width": null
       }
      },
-     "cafce154ce254e2489b3c3231cd3afc8": {
-      "model_module": "@jupyter-widgets/base",
+     "c43b6d1b7dc04e8ea921c5940f890126": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "HTMLModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_4a23001ae4824747ac51dbe487a0a126",
+       "placeholder": "​",
+       "style": "IPY_MODEL_add5c8f3c40642ef8e4cf96579325ec3",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 124/124 [00:00&lt;00:00, 24.5kB/s]"
       }
      },
-     "cb2bcca49ce84b5f813a3e09a28e17e8": {
+     "c655320743a4421684aca235370c46c1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23025,39 +22942,168 @@
        "width": null
       }
      },
-     "ccd9adc54d384d26a6e04c69df253074": {
+     "c9ac7e38eb534678896df289998d1855": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_b9ffe90723d54a178c02c4a71079ef4f",
+        "IPY_MODEL_6b5428d552484be2b498aa06f2cd12d7",
+        "IPY_MODEL_d8504344e45a40dba0505fafd0a4fef8"
+       ],
+       "layout": "IPY_MODEL_c655320743a4421684aca235370c46c1",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "d096417d88bf4cc79562f5b69c150050": {
+     "c9d427e21ae84197b7ff8cdb8bb4c89d": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "FloatProgressModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_860eac54afd644cdb66576f10d1c4b94",
+       "max": 1355256.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_b2dd202e54cc4c4bbf18779cef088e63",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1355256.0
+      }
+     },
+     "cf0868984e46408fb4a1d566ec3db0b0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_aa33f2b3f7af4a5c956ebdbae08dabec",
+        "IPY_MODEL_c9d427e21ae84197b7ff8cdb8bb4c89d",
+        "IPY_MODEL_0dc2fdfccac3440ea6a358b4b727da08"
+       ],
+       "layout": "IPY_MODEL_bec18b05c0a8413485c9f6fb6d8d51dd",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "d2847e7beeb442018670534e35889459": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "d3bf56862e814e46a08ce4949ce5df67": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_621583ed869b478d913fc04a87073a95",
+       "placeholder": "​",
+       "style": "IPY_MODEL_d2847e7beeb442018670534e35889459",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "merges.txt: 100%"
       }
      },
-     "d310211006ce4cb6aac419247e69239f": {
+     "d8504344e45a40dba0505fafd0a4fef8": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_4647f9260ab248d7a350e7a07c999a9b",
+       "placeholder": "​",
+       "style": "IPY_MODEL_7f602aa1ed364df1a417f0cf429e4a5c",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 25.8MB/s]"
+      }
+     },
+     "ddeeebc813c54775ab8841a88e512534": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_1dffcf97acf041dfa4829930fc637e56",
+       "placeholder": "​",
+       "style": "IPY_MODEL_bf988798a38d454fa0c118280dbeab56",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 548M/548M [00:02&lt;00:00, 243MB/s]"
+      }
+     },
+     "de2a35b1b3264fd2ac593af14da5b645": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23110,31 +23156,43 @@
        "width": null
       }
      },
-     "d682c8c5f56443c38680f259b4f0faef": {
+     "e49babdcc96f4f99af8dba7cd03202fc": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_5604e63f816e45f280635edb76818789",
-        "IPY_MODEL_bba7279dfd574dd5b910973cde0fc709",
-        "IPY_MODEL_d8633f28258d46318664248a3566c7f3"
-       ],
-       "layout": "IPY_MODEL_4ab50487fcca41b59f1392f90cdc6757",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "d811ce6fc2d94b5a8b4a8e4bb7b04432": {
+     "ed04372d5ce8433d848d96e975723537": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "f06d8ed2f2fc40c78e6f71a980042fa2": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23187,30 +23245,43 @@
        "width": null
       }
      },
-     "d8633f28258d46318664248a3566c7f3": {
+     "f6e7f731edb44410a1bbe854b20b75b3": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_7f9b1c85b4da4b068ea5593efc94d962",
-       "placeholder": "​",
-       "style": "IPY_MODEL_68b77f75328e4a539b09b13b20cac43b",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 4.12MB/s]"
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "e79d7678437a45a39ca514276be1e4a9": {
+     "f988b1609264444ea2e348b491a028a4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "fe14fba431ee44719e467a94d38602c1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23262,77 +23333,6 @@
        "visibility": null,
        "width": null
       }
-     },
-     "ea70744b24bb4f8797907ee24a3e69c9": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_152fb19b656b444982eb11fae36392ae",
-        "IPY_MODEL_4ad663a579114d4eb47fd1364f66ccc0",
-        "IPY_MODEL_ef653dc2d7684ec5984b824ab60c017e"
-       ],
-       "layout": "IPY_MODEL_729153cf51e244f598470b0f440f9b25",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "ef653dc2d7684ec5984b824ab60c017e": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_822ddcdd61b443ce9bc565addafb6984",
-       "placeholder": "​",
-       "style": "IPY_MODEL_90da3925110a458aad1cc1c6f8f98fd4",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 4.95kB/s]"
-      }
-     },
-     "f2a3e2ebfaad4d4d8b38096dac3f0a14": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_9987dd8fb4df47f8bad2480d55f0644e",
-        "IPY_MODEL_3328cd9e412b4e8ea246fbb324d064c2",
-        "IPY_MODEL_5ce85280f03e416599efda8942bfd6b7"
-       ],
-       "layout": "IPY_MODEL_77db344030f64e1cb37bde70ed8e7676",
-       "tabbable": null,
-       "tooltip": null
-      }
      }
     },
     "version_major": 2,
diff --git a/generated/demos/Main_Demo.html b/generated/demos/Main_Demo.html
index 655e17469..6700ea437 100644
--- a/generated/demos/Main_Demo.html
+++ b/generated/demos/Main_Demo.html
@@ -385,11 +385,11 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-4c03d420-fc38" style="margin: 15px 0;"/>
+<div id="circuits-vis-48c8c518-76ab" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, Hello } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-4c03d420-fc38",
+      "circuits-vis-48c8c518-76ab",
       Hello,
       {"name": "Neel"}
     )
@@ -440,7 +440,7 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
-&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7fcdda667510&gt;
+&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7f0f50f9df90&gt;
 </pre></div></div>
 </div>
 <p>Plotting helper functions:</p>
@@ -593,16 +593,30 @@ <h2>Caching all Activations<a class="headerlink" href="#Caching-all-Activations"
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-02b5f1a2-ff46" style="margin: 15px 0;"/>
+<div id="circuits-vis-ac0a9437-0fc4" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-02b5f1a2-ff46",
+      "circuits-vis-ac0a9437-0fc4",
       AttentionPatterns,
       {"tokens": ["<|endoftext|>", "Natural", " language", " processing", " tasks", ",", " such", " as", " question", " answering", ",", " machine", " translation", ",", " reading", " comprehension", ",", " and", " summar", "ization", ",", " are", " typically", " approached", " with", " supervised", " learning", " on", " tasks", "pe", "cific", " datasets", "."], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}
     )
     </script></div>
 </div>
+<p>In this case, we only wanted the layer 0 attention patterns, but we are storing the internal activations from all locations in the model. It’s convenient to have access to all activations, but this can be prohibitively expensive for memory use with larger models, batch sizes, or sequence lengths. In addition, we don’t need to do the full forward pass through the model to collect layer 0 attention patterns. The following cell will collect only the layer 0 attention patterns and stop the forward
+pass at layer 1, requiring far less memory and compute.</p>
+<div class="nbinput nblast docutils container">
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[14]:
+</pre></div>
+</div>
+<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">attn_hook_name</span> <span class="o">=</span> <span class="s2">&quot;blocks.0.attn.hook_pattern&quot;</span>
+<span class="n">attn_layer</span> <span class="o">=</span> <span class="mi">0</span>
+<span class="n">_</span><span class="p">,</span> <span class="n">gpt2_attn_cache</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">run_with_cache</span><span class="p">(</span><span class="n">gpt2_tokens</span><span class="p">,</span> <span class="n">remove_batch_dim</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">stop_at_layer</span><span class="o">=</span><span class="n">attn_layer</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">names_filter</span><span class="o">=</span><span class="p">[</span><span class="n">attn_hook_name</span><span class="p">])</span>
+<span class="n">gpt2_attn</span> <span class="o">=</span> <span class="n">gpt2_attn_cache</span><span class="p">[</span><span class="n">attn_hook_name</span><span class="p">]</span>
+<span class="k">assert</span> <span class="n">torch</span><span class="o">.</span><span class="n">equal</span><span class="p">(</span><span class="n">gpt2_attn</span><span class="p">,</span> <span class="n">attention_pattern</span><span class="p">)</span>
+</pre></div>
+</div>
+</div>
 </section>
 <section id="Hooks:-Intervening-on-Activations">
 <h2>Hooks: Intervening on Activations<a class="headerlink" href="#Hooks:-Intervening-on-Activations" title="Permalink to this heading">#</a></h2>
@@ -617,7 +631,7 @@ <h2>Hooks: Intervening on Activations<a class="headerlink" href="#Hooks:-Interve
 <p>We define a <code class="docutils literal notranslate"><span class="pre">head_ablation_hook</span></code> function. This takes the value tensor for attention layer 0, and sets the component with <code class="docutils literal notranslate"><span class="pre">head_index==7</span></code> to zero and returns it (Note - we return by convention, but since we’re editing the activation in-place, we don’t strictly <em>need</em> to).</p>
 <p>We then use the <code class="docutils literal notranslate"><span class="pre">run_with_hooks</span></code> helper function to run the model and <em>temporarily</em> add in the hook for just this run. We enter in the hook as a tuple of the activation name (also the hook point name - found with <code class="docutils literal notranslate"><span class="pre">utils.get_act_name</span></code>) and the hook function.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[14]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[15]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">layer_to_ablate</span> <span class="o">=</span> <span class="mi">0</span>
@@ -669,7 +683,7 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <p>Here, our clean prompt is “After John and Mary went to the store, <strong>Mary</strong> gave a bottle of milk to”, our corrupted prompt is “After John and Mary went to the store, <strong>John</strong> gave a bottle of milk to”, and our metric is the difference between the correct logit ( John) and the incorrect logit ( Mary) on the final token.</p>
 <p>We see that the logit difference is significantly positive on the clean prompt, and significantly negative on the corrupted prompt, showing that the model is capable of doing the task!</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[15]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[16]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">clean_prompt</span> <span class="o">=</span> <span class="s2">&quot;After John and Mary went to the store, Mary gave a bottle of milk to&quot;</span>
@@ -709,7 +723,7 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <p>We now setup the hook function to do activation patching. Here, we’ll patch in the <a class="reference external" href="https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=DHp9vZ0h9lA9OCrzG2Y3rrzH">residual stream</a> at the start of a specific layer and at a specific position. This will let us see how much the model is using the residual stream at that layer and position to represent the key information for the task.</p>
 <p>We want to iterate over all layers and positions, so we write the hook to take in an position parameter. Hook functions must have the input signature (activation, hook), but we can use <code class="docutils literal notranslate"><span class="pre">functools.partial</span></code> to set the position parameter before passing it to <code class="docutils literal notranslate"><span class="pre">run_with_hooks</span></code></p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[16]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[17]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># We define a residual stream patching hook</span>
@@ -748,11 +762,11 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "8886699251a949f49e757b9fdc7ca890"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ddd91d21a45e49a8a90fd6693fc0fe90"}</script></div>
 </div>
 <p>We can now visualize the results, and see that this computation is extremely localised within the model. Initially, the second subject (Mary) token is all that matters (naturally, as it’s the only different token), and all relevant information remains here until heads in layer 7 and 8 move this to the final token where it’s used to predict the indirect object. (Note - the heads are in layer 7 and 8, not 8 and 9, because we patched in the residual stream at the <em>start</em> of each layer)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[17]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[18]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Add the index to the end of the label, because plotly doesn&#39;t like duplicate labels</span>
@@ -769,9 +783,9 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="9a4c3889-fbfe-43e2-a51a-d3c551c88f8d" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("9a4c3889-fbfe-43e2-a51a-d3c551c88f8d")) {                    Plotly.newPlot(                        "9a4c3889-fbfe-43e2-a51a-d3c551c88f8d",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="a86e9b27-7dd3-47fb-9141-10fa1ea8195f" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a86e9b27-7dd3-47fb-9141-10fa1ea8195f")) {                    Plotly.newPlot(                        "a86e9b27-7dd3-47fb-9141-10fa1ea8195f",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('9a4c3889-fbfe-43e2-a51a-d3c551c88f8d');
+var gd = document.getElementById('a86e9b27-7dd3-47fb-9141-10fa1ea8195f');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -808,7 +822,7 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <p>To see why this is important, let’s say that the model is trying to predict the next token in a news article about Michael Jordan. The token “ Michael”, in general, could be followed by many surnames. But an induction head will look from that occurrence of “ Michael” to the token after previous occurrences of “ Michael”, ie “ Jordan” and can confidently predict that that will come next.</p>
 <p>An interesting fact about induction heads is that they generalise to arbitrary sequences of repeated tokens. We can see this by generating sequences of 50 random tokens, repeated twice, and plotting the average loss at predicting the next token, by position. We see that the model goes from terrible to very good at the halfway point.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[18]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[19]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">10</span>
@@ -833,9 +847,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b221d135-41b1-42fe-b935-2c48e6f5a584" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b221d135-41b1-42fe-b935-2c48e6f5a584")) {                    Plotly.newPlot(                        "b221d135-41b1-42fe-b935-2c48e6f5a584",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.930997848510742,12.973093032836914,14.311477661132812,13.683721542358398,13.431002616882324,12.055716514587402,13.882497787475586,12.124085426330566,10.646708488464355,14.250656127929688,12.821611404418945,12.736834526062012,11.802390098571777,11.932378768920898,12.010724067687988,12.102657318115234,11.457335472106934,11.689569473266602,11.127102851867676,12.51093864440918,10.938901901245117,11.951817512512207,11.50601863861084,11.922638893127441,11.712303161621094,11.965950012207031,11.912420272827148,11.779227256774902,12.358148574829102,11.38554573059082,12.330202102661133,10.471684455871582,11.231712341308594,11.510209083557129,11.543947219848633,11.340058326721191,10.549806594848633,12.171330451965332,11.418136596679688,10.7109375,11.285449981689453,10.970927238464355,12.132821083068848,12.104890823364258,11.697572708129883,12.078733444213867,11.500871658325195,11.366061210632324,11.200624465942383,11.578920364379883,3.00230073928833,0.9546955227851868,0.7389153838157654,0.5628250241279602,0.4280739426612854,0.3580504059791565,0.5007835626602173,0.3580631613731384,0.24043424427509308,0.4543682038784027,0.5478570461273193,0.13366922736167908,0.13654688000679016,0.23078858852386475,0.1546991914510727,0.05507778376340866,0.35659217834472656,0.07221563160419464,0.050734974443912506,0.29230499267578125,0.04061192646622658,0.1435699164867401,0.3380035161972046,0.06990544497966766,0.18574923276901245,0.05935591459274292,0.08002041280269623,0.11634248495101929,0.09985627233982086,0.08987277746200562,0.04109250754117966,0.028509577736258507,0.052600979804992676,0.04117293283343315,0.08193311840295792,0.02876918390393257,0.011056417599320412,0.10547198355197906,0.13078591227531433,0.1317104995250702,0.033281438052654266,0.02778303623199463,0.03573056310415268,0.047907400876283646,0.0208450797945261,0.1270831674337387,0.07031372934579849,0.06472618877887726,0.0732506588101387],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc")) {                    Plotly.newPlot(                        "d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.934269905090332,14.23181438446045,11.729150772094727,15.214762687683105,13.592933654785156,14.211908340454102,13.205050468444824,13.07355785369873,11.535038948059082,12.773419380187988,13.354753494262695,12.027746200561523,12.879249572753906,11.845925331115723,12.831506729125977,11.792170524597168,12.737360000610352,12.617140769958496,12.619315147399902,12.312422752380371,11.479436874389648,12.757272720336914,11.545267105102539,12.173948287963867,11.990314483642578,11.905129432678223,11.678251266479492,11.099139213562012,10.786980628967285,12.338418960571289,11.62226676940918,11.612360000610352,10.657169342041016,11.615835189819336,11.122636795043945,10.654256820678711,11.861252784729004,11.142145156860352,11.718962669372559,11.342132568359375,11.637442588806152,11.20296859741211,12.066685676574707,10.65315055847168,11.302213668823242,11.483949661254883,10.655682563781738,11.83389949798584,11.766695976257324,11.974489212036133,3.564096450805664,1.2277376651763916,0.9820038676261902,1.200751543045044,0.4091821312904358,0.5843209624290466,0.3591119647026062,0.43036898970603943,0.24223056435585022,0.44049954414367676,0.41071295738220215,0.21330496668815613,0.15890493988990784,0.31460171937942505,0.08366963267326355,0.2029363214969635,0.17958413064479828,0.1480543464422226,0.034813292324543,0.06144014745950699,0.09302718937397003,0.06429728120565414,0.4040629267692566,0.12488675117492676,0.08299500495195389,0.02960968390107155,0.01776202768087387,0.03346758708357811,0.10293184220790863,0.0828002542257309,0.008384309709072113,0.031925592571496964,0.1624717116355896,0.05249609425663948,0.02139073610305786,0.03643417730927467,0.07777489721775055,0.1070786863565445,0.03805286064743996,0.11206188052892685,0.1607268750667572,0.13429032266139984,0.16408373415470123,0.05713384598493576,0.12058018147945404,0.09626038372516632,0.04750080779194832,0.1064792275428772,0.15790846943855286],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('b221d135-41b1-42fe-b935-2c48e6f5a584');
+var gd = document.getElementById('d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -874,7 +888,7 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 </li>
 </ul>
 </details><div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[19]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[20]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># We make a tensor to store the induction score for each head. We put it on the model&#39;s device to avoid needing to move things between the GPU and CPU, which can be slow.</span>
@@ -915,9 +929,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="f61e6828-f047-4b78-be4c-94c6a5a16b3b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f61e6828-f047-4b78-be4c-94c6a5a16b3b")) {                    Plotly.newPlot(                        "f61e6828-f047-4b78-be4c-94c6a5a16b3b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010005488060414791,0.00010679646220523864,0.011149473488330841,1.6619707139398088e-06,0.00023922690888866782,0.00015111741959117353,0.009801512584090233,0.0007387498626485467,0.009316714480519295,0.009042520076036453,0.006868044380098581,0.01548066921532154],[0.0008192680543288589,0.00046259158989414573,0.0020540570840239525,0.014562157914042473,0.004917391575872898,0.01147772278636694,0.01584085077047348,0.01319159660488367,0.012704173102974892,0.015902359038591385,0.006454707123339176,0.0005202066386118531],[0.005110130179673433,0.018723005428910255,0.0012636644532904029,0.0013060116907581687,0.013963320292532444,0.0018745333654806018,0.004315933212637901,0.008237375877797604,0.0035762235056608915,0.0011971687199547887,0.0006931528332643211,0.010485384613275528],[0.015022697858512402,0.0073833102360367775,0.0018157829763367772,0.01069782767444849,0.021013276651501656,0.012958213686943054,0.001296360045671463,0.0009169265395030379,0.004911302588880062,0.011581930331885815,0.007676774635910988,0.00595051096752286],[0.015315228141844273,0.01408644113689661,0.014493301510810852,0.009663796983659267,0.018403636291623116,0.013842382468283176,0.00839169230312109,0.0018653222359716892,0.016032662242650986,0.013998506590723991,0.018994230777025223,7.747924080092616e-09],[0.4260586202144623,0.8949548602104187,0.01510448008775711,0.007229045499116182,0.012087161652743816,0.9142396450042725,0.01206196192651987,0.01813812553882599,0.02615417167544365,0.027872497215867043,0.018886346369981766,0.017591215670108795],[0.01042243279516697,0.016652364283800125,0.01844092272222042,0.014439660124480724,0.023017292842268944,0.011790945194661617,0.029514683410525322,0.010329627431929111,0.011278615333139896,0.904080867767334,0.03695948049426079,0.013053759932518005],[0.010848619975149632,0.18712441623210907,0.8447970151901245,0.018071545287966728,0.01874406263232231,0.01866166852414608,0.04627839848399162,0.09087574481964111,0.017286069691181183,0.019067486748099327,0.9059284329414368,0.05858321860432625],[0.017044229432940483,0.39027440547943115,0.015756050124764442,0.04679980129003525,0.018212294206023216,0.013268682174384594,0.1567089557647705,0.013300132006406784,0.0325496643781662,0.032548386603593826,0.06684791296720505,0.019402673467993736],[0.253298819065094,0.18304164707660675,0.10468680411577225,0.012513653375208378,0.0902019739151001,0.026693832129240036,0.4520256519317627,0.029920928180217743,0.05179687589406967,0.476281076669693,0.01758650131523609,0.04082362726330757],[0.345258504152298,0.5076920390129089,0.0380970723927021,0.14610175788402557,0.06365255266427994,0.01531550195068121,0.2981257736682892,0.48563432693481445,0.04946523532271385,0.016827359795570374,0.1584998071193695,0.2598787248134613],[0.017348697409033775,0.05146760120987892,0.03453077748417854,0.009557337500154972,0.0365176685154438,0.09732990711927414,0.0516226589679718,0.06467494368553162,0.008705069310963154,0.30858248472213745,0.38851720094680786,0.02194729819893837]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b2a3ed8a-ba81-4aac-a1a1-60e16774f655" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b2a3ed8a-ba81-4aac-a1a1-60e16774f655")) {                    Plotly.newPlot(                        "b2a3ed8a-ba81-4aac-a1a1-60e16774f655",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010606949217617512,0.00013902947830501944,0.01081207673996687,5.408500101111713e-07,0.00022116937907412648,7.771956006763503e-05,0.00958997942507267,0.0009253817261196673,0.009193798527121544,0.009053227491676807,0.006911635398864746,0.014761571772396564],[0.0008132570656016469,0.0004268861666787416,0.002124645747244358,0.01425299420952797,0.005162382964044809,0.010268471203744411,0.015934234485030174,0.013339280150830746,0.012906215153634548,0.016355415806174278,0.006486278492957354,0.0005485203000716865],[0.004910458344966173,0.019308894872665405,0.0012424595188349485,0.0010258526308462024,0.013039778918027878,0.0015818236861377954,0.00429101288318634,0.008304944261908531,0.003105038544163108,0.000971563218627125,0.0007218025857582688,0.010301434434950352],[0.015241926535964012,0.006064379122108221,0.00234038638882339,0.01002125907689333,0.021940935403108597,0.01423431932926178,0.0017911610193550587,0.0017028121510520577,0.005022364668548107,0.012421459890902042,0.00839681550860405,0.005942116491496563],[0.015156026929616928,0.013609741814434528,0.01541218627244234,0.00972918700426817,0.018237268552184105,0.013402285054326057,0.009189526550471783,0.0018425184534862638,0.015492293052375317,0.013836968690156937,0.0181787870824337,3.1250738885546525e-08],[0.42406418919563293,0.9055006504058838,0.015690578147768974,0.01017544511705637,0.01198639441281557,0.9278296828269958,0.012438608333468437,0.018412206321954727,0.027197834104299545,0.028249241411685944,0.019754743203520775,0.018243789672851562],[0.012461011298000813,0.01634187251329422,0.019040238112211227,0.015691598877310753,0.02333749644458294,0.012633594684302807,0.029116196557879448,0.012918342836201191,0.009929743595421314,0.92023104429245,0.035106346011161804,0.015322730876505375],[0.011738991364836693,0.17485056817531586,0.8415674567222595,0.01858351193368435,0.018508559092879295,0.018142390996217728,0.04514370858669281,0.08067239820957184,0.017961328849196434,0.019256191328167915,0.9187174439430237,0.052430037409067154],[0.01744176633656025,0.3839356303215027,0.01593034528195858,0.0451587438583374,0.018434638157486916,0.01456737332046032,0.14198516309261322,0.014199082739651203,0.03386015444993973,0.0334649458527565,0.06564274430274963,0.02554652839899063],[0.2423967868089676,0.19136165082454681,0.1043233796954155,0.012038059532642365,0.0845528319478035,0.027191946282982826,0.4313036799430847,0.028380122035741806,0.04645952582359314,0.4677841365337372,0.017513224855065346,0.03886174410581589],[0.3296402394771576,0.48611578345298767,0.0376255176961422,0.14191953837871552,0.06014999374747276,0.01589331030845642,0.28858682513237,0.479020357131958,0.05016819015145302,0.015279495157301426,0.14848238229751587,0.25642627477645874],[0.01643357053399086,0.05329973250627518,0.035110954195261,0.011042965576052666,0.03575966879725456,0.10564529150724411,0.048889338970184326,0.06571993976831436,0.009506475180387497,0.30028679966926575,0.3763805329799652,0.025962304323911667]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f61e6828-f047-4b78-be4c-94c6a5a16b3b');
+var gd = document.getElementById('b2a3ed8a-ba81-4aac-a1a1-60e16774f655');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -946,7 +960,7 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <p>Head 5 in Layer 5 scores extremely highly on this score, and we can feed in a shorter repeated random sequence, visualize the attention pattern for it and see this directly - including the “induction stripe” at <code class="docutils literal notranslate"><span class="pre">seq_len-1</span></code> tokens back.</p>
 <p>This time we put in a hook on the attention pattern activation to visualize the pattern of the relevant head.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[20]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[21]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">IN_GITHUB</span><span class="p">:</span>
@@ -985,11 +999,11 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-ec824f3e-cfa6" style="margin: 15px 0;"/>
+<div id="circuits-vis-61a73e43-0cbc" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-ec824f3e-cfa6",
+      "circuits-vis-61a73e43-0cbc",
       AttentionPatterns,
       {"tokens": ["use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44", "use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44"], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}
     )
@@ -1003,7 +1017,7 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <p><strong>Note:</strong> TransformerLens does not currently support multi-GPU models (which you want for models above eg 7B parameters), but this feature is coming soon!</p>
 <p>Notably, this means that analysis can be near immediately re-run on a different model by just changing the name - to see this, let’s load in DistilGPT-2 (a distilled version of GPT-2, with half as many layers) and copy the code from above to see the induction heads in that model.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[21]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[22]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># NBVAL_IGNORE_OUTPUT</span>
@@ -1015,37 +1029,37 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "504b41851a444c31b688e5e8b063add5"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2b653bed330e47bfaf1eb97604ffb148"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "8efd0c3ea3bc4e5d8c1b966bff063c3a"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "2f95bda34e124bdb8ad42b97968c2560"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "5df370f098ea41fba5fb462b17adbe25"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "1bfe4dc8f9d64ed3a70a0e6e41b61d34"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "d5f0c8842a284fdda4dfa3206605c35f"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "363a53f62bad40f38d9b57415fcdd27a"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "a678c766e93d424daa0150a52a8f38a2"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "baa802cac1bf4a77bd2d6f2da4e0c206"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "da7af55f6bb5479b852a41df346f73ee"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "85239cb8d6ba4d288b001290e919d816"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -1056,7 +1070,7 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 </pre></div></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[22]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[23]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><br/><span></span><span class="c1"># We make a tensor to store the induction score for each head. We put it on the model&#39;s device to avoid needing to move things between the GPU and CPU, which can be slow.</span>
@@ -1097,9 +1111,9 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2b76182f-a588-4611-91f1-3158be36fbea" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2b76182f-a588-4611-91f1-3158be36fbea")) {                    Plotly.newPlot(                        "2b76182f-a588-4611-91f1-3158be36fbea",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.009826192632317543,0.00015374009672086686,0.012172531336545944,6.603482688660733e-06,0.000805103627499193,2.0451569071155973e-05,0.008835355751216412,0.0015585398068651557,0.008454771712422371,0.009232211858034134,0.009152095764875412,0.016232741996645927],[0.0028829516377300024,0.01789030432701111,0.0022680708207190037,0.0004275149549357593,0.012768547050654888,0.0016649786848574877,0.0051553091034293175,0.015345298685133457,0.002696307608857751,8.173943206202239e-05,0.003935561515390873,0.014916213229298592],[0.007932284846901894,0.006039753090590239,0.012212091125547886,0.0025668356101959944,0.019200311973690987,0.004120599944144487,0.009461229667067528,0.0011959228431805968,0.017644984647631645,0.013755349442362785,0.021033549681305885,6.036892652439008e-10],[0.006974583957344294,0.22314414381980896,0.8409926295280457,0.01541733369231224,0.017725780606269836,0.014554478228092194,0.019251752644777298,0.18048787117004395,0.016629252582788467,0.015987560153007507,0.9273980855941772,0.46241772174835205],[0.2584259510040283,0.21280966699123383,0.0862935483455658,0.011735445819795132,0.07724905014038086,0.02455822005867958,0.6039906144142151,0.02535823918879032,0.06264935433864594,0.6323530077934265,0.017074570059776306,0.055065713822841644],[0.021564770489931107,0.07452036440372467,0.052915140986442566,0.011579645797610283,0.034101374447345734,0.17451111972332,0.07449180632829666,0.08992660045623779,0.008418177254498005,0.43939897418022156,0.17041026055812836,0.02964613400399685]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e151a52d-4559-414d-bbd0-4a08a83b57c6" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e151a52d-4559-414d-bbd0-4a08a83b57c6")) {                    Plotly.newPlot(                        "e151a52d-4559-414d-bbd0-4a08a83b57c6",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010150859132409096,0.00021067557099740952,0.011675543151795864,5.645640158036258e-06,0.0009093668777495623,1.0044651389762294e-05,0.008776976726949215,0.0014797636540606618,0.008503900840878487,0.009458577260375023,0.009176097810268402,0.015341502614319324],[0.0030316568445414305,0.018613651394844055,0.003521384671330452,0.0002105814783135429,0.012771016918122768,0.002262213034555316,0.005199276376515627,0.015398663468658924,0.001596646150574088,7.471267599612474e-05,0.004113694187253714,0.014634193852543831],[0.00835011899471283,0.005591983906924725,0.01345061045140028,0.0016680879052728415,0.019946973770856857,0.004370617680251598,0.009695201180875301,0.001158412080258131,0.01791170984506607,0.01292894221842289,0.021461093798279762,5.112248818717191e-11],[0.005671565420925617,0.20814286172389984,0.8488407135009766,0.01734151318669319,0.017860308289527893,0.014111736789345741,0.020909568294882774,0.1804729551076889,0.01673540472984314,0.0179011020809412,0.9288043975830078,0.4491482973098755],[0.26883465051651,0.23360644280910492,0.08689460158348083,0.011528643779456615,0.07701855152845383,0.024650055915117264,0.6229627132415771,0.025258244946599007,0.06073903664946556,0.6466246247291565,0.017515525221824646,0.054975464940071106],[0.021400248631834984,0.07713880389928818,0.05230269953608513,0.013922988437116146,0.032270368188619614,0.18037253618240356,0.07306526601314545,0.0951840952038765,0.007617602590471506,0.4484187066555023,0.17469297349452972,0.03143536299467087]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('2b76182f-a588-4611-91f1-3158be36fbea');
+var gd = document.getElementById('e151a52d-4559-414d-bbd0-4a08a83b57c6');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1233,7 +1247,7 @@ <h3>Parameter Names<a class="headerlink" href="#Parameter-Names" title="Permalin
 * <code class="docutils literal notranslate"><span class="pre">n_ctx</span></code>: 1024. The maximum number of tokens in an input prompt.</p>
 <p><strong>Transformer Block parameters:</strong> Replace 0 with the relevant layer index.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[23]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[24]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
@@ -1263,7 +1277,7 @@ <h3>Parameter Names<a class="headerlink" href="#Parameter-Names" title="Permalin
 </div>
 <p><strong>Embedding &amp; Unembedding parameters:</strong></p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[24]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[25]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
@@ -1292,7 +1306,7 @@ <h3>Activation + Hook Names<a class="headerlink" href="#Activation-+-Hook-Names"
 is the LayerNorm before the unembed.</p>
 <p>Note 2: <em>Every</em> activation apart from the attention pattern and attention scores has shape beginning with <code class="docutils literal notranslate"><span class="pre">[batch,</span> <span class="pre">position]</span></code>. The attention pattern and scores have shape <code class="docutils literal notranslate"><span class="pre">[batch,</span> <span class="pre">head_index,</span> <span class="pre">dest_position,</span> <span class="pre">source_position]</span></code> (the numbers are the same, unless we’re using caching).</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[25]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[26]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">test_prompt</span> <span class="o">=</span> <span class="s2">&quot;The quick brown fox jumped over the lazy dog&quot;</span>
@@ -1357,7 +1371,7 @@ <h3>Folding LayerNorm (For the Curious)<a class="headerlink" href="#Folding-Laye
 <p>A fun consequence of LayerNorm folding is that it creates a bias across the unembed, a <code class="docutils literal notranslate"><span class="pre">d_vocab</span></code> length vector that is added to the output logits - GPT-2 is not trained with this, but it <em>is</em> trained with a final LayerNorm that contains a bias.</p>
 <p>Turns out, this LayerNorm bias learns structure of the data that we can only see after folding! In particular, it essentially learns <strong>unigram statistics</strong> - rare tokens get suppressed, common tokens get boosted, by pretty dramatic degrees! Let’s list the top and bottom 20 - at the top we see common punctuation and words like “ the” and “ and”, at the bottom we see weird-ass tokens like “ RandomRedditor”:</p>
 <div class="nbinput nblast docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[26]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[27]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">unembed_bias</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">unembed</span><span class="o">.</span><span class="n">b_U</span>
@@ -1366,7 +1380,7 @@ <h3>Folding LayerNorm (For the Curious)<a class="headerlink" href="#Folding-Laye
 </div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[27]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[28]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">top_k</span> <span class="o">=</span> <span class="mi">20</span>
@@ -1433,7 +1447,7 @@ <h3>Folding LayerNorm (For the Curious)<a class="headerlink" href="#Folding-Laye
 </div>
 <p>This can have real consequences for interpretability - for example, this bias favours “ John” over “ Mary” by about 1.2, about 1/3 of the effect size of the Indirect Object Identification Circuit! All other things being the same, this makes the John token 3.6x times more likely than the Mary token.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[28]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[29]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">john_bias</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">unembed</span><span class="o">.</span><span class="n">b_U</span><span class="p">[</span><span class="n">model</span><span class="o">.</span><span class="n">to_single_token</span><span class="p">(</span><span class="s1">&#39; John&#39;</span><span class="p">)]</span>
@@ -1470,7 +1484,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 <p>Some observations - there are a lot of arbitrary-ish details in here! * The tokenizer splits on spaces, so no token contains two words. * Tokens include the preceding space, and whether the first token is a capital letter. <code class="docutils literal notranslate"><span class="pre">how</span></code> and <code class="docutils literal notranslate"><span class="pre">how</span></code> are different tokens! * Common words are single tokens, even if fairly long (<code class="docutils literal notranslate"><span class="pre">paragraph</span></code>) while uncommon words are split into multiple tokens (<code class="docutils literal notranslate"><span class="pre">token|ized</span></code>). * Tokens <em>mostly</em> split on punctuation characters (eg <code class="docutils literal notranslate"><span class="pre">*</span></code> and <code class="docutils literal notranslate"><span class="pre">.</span></code>), but eg <code class="docutils literal notranslate"><span class="pre">'s</span></code> is a
 single token.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[29]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[30]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">example_text</span> <span class="o">=</span> <span class="s2">&quot;The first thing you need to figure out is *how* things are tokenized. `model.to_str_tokens` splits a string into the tokens *as a list of substrings*, and so lets you explore what the text looks like. To demonstrate this, let&#39;s use it on this paragraph.&quot;</span>
@@ -1489,7 +1503,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </div>
 <p>The transformer needs to take in a sequence of integers, not strings, so we need to convert these tokens into integers. <code class="docutils literal notranslate"><span class="pre">model.to_tokens</span></code> does this, and returns a tensor of integers on the model’s device (shape <code class="docutils literal notranslate"><span class="pre">[batch,</span> <span class="pre">position]</span></code>). It maps a string to a batch of size 1.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[30]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[31]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">example_text_tokens</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">to_tokens</span><span class="p">(</span><span class="n">example_text</span><span class="p">)</span>
@@ -1514,7 +1528,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 <p><code class="docutils literal notranslate"><span class="pre">to_tokens</span></code> can also take in a list of strings, and return a batch of size <code class="docutils literal notranslate"><span class="pre">len(strings)</span></code>. If the strings are different numbers of tokens, it adds a PAD token to the end of the shorter strings to make them the same length.</p>
 <p>(Note: In GPT-2, 50256 signifies both the beginning of sequence, end of sequence and padding token - see the <code class="docutils literal notranslate"><span class="pre">prepend_bos</span></code> section for details)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[31]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[32]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">example_multi_text</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;The cat sat on the mat.&quot;</span><span class="p">,</span> <span class="s2">&quot;The cat sat on the mat really hard.&quot;</span><span class="p">]</span>
@@ -1539,7 +1553,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 <p>Note further that the log probs have shape <code class="docutils literal notranslate"><span class="pre">[batch,</span> <span class="pre">position,</span> <span class="pre">d_vocab]==[1,</span> <span class="pre">8,</span> <span class="pre">50257]</span></code>, with a vector of log probs predicting the next token for <em>every</em> token position. GPT-2 uses causal attention which means heads can only look backwards (equivalently, information can only move forwards in the model.), so the log probs at position k are only a function of the first k tokens, and it can’t just cheat and look at the k+1 th token. This structure lets it generate text more efficiently, and lets
 it treat every <em>token</em> as a training example, rather than every <em>sequence</em>.</p>
 </details><div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[32]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[33]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">cat_text</span> <span class="o">=</span> <span class="s2">&quot;The cat sat on the mat.&quot;</span>
@@ -1552,19 +1566,26 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </pre></div>
 </div>
 </div>
-<div class="nboutput nblast docutils container">
+<div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
 Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])
+</pre></div></div>
+</div>
+<div class="nboutput nblast docutils container">
+<div class="prompt empty docutils container">
+</div>
+<div class="output_area docutils container">
+<div class="highlight"><pre>
 | The| probability: 11.98%
 </pre></div></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">model.to_string</span></code> is the inverse of <code class="docutils literal notranslate"><span class="pre">to_tokens</span></code> and maps a tensor of integers to a string or list of strings. It also works on integers and lists of integers.</p>
 <p>For example, let’s look up token 256 (due to technical details of tokenization, this will be the most common pair of ASCII characters!), and also verify that our tokens above map back to a string.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[33]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[34]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Token 256 - the most common pair of ASCII characters: |</span><span class="si">{</span><span class="n">model</span><span class="o">.</span><span class="n">to_string</span><span class="p">(</span><span class="mi">256</span><span class="p">)</span><span class="si">}</span><span class="s2">|&quot;</span><span class="p">)</span>
@@ -1587,7 +1608,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 <p>A related annoyance of tokenization is that it’s hard to figure out how many tokens a string will break into. <code class="docutils literal notranslate"><span class="pre">model.get_token_position(single_token,</span> <span class="pre">tokens)</span></code> returns the position of <code class="docutils literal notranslate"><span class="pre">single_token</span></code> in <code class="docutils literal notranslate"><span class="pre">tokens</span></code>. <code class="docutils literal notranslate"><span class="pre">tokens</span></code> can be either a string or a tensor of tokens.</p>
 <p>Note that position is zero-indexed, it’s two (ie third) because there’s a beginning of sequence token automatically prepended (see the next section for details)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[34]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[35]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;With BOS:&quot;</span><span class="p">,</span> <span class="n">model</span><span class="o">.</span><span class="n">get_token_position</span><span class="p">(</span><span class="s2">&quot; cat&quot;</span><span class="p">,</span> <span class="s2">&quot;The cat sat on the mat&quot;</span><span class="p">))</span>
@@ -1606,7 +1627,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </div>
 <p>If there are multiple copies of the token, we can set <code class="docutils literal notranslate"><span class="pre">mode=&quot;first&quot;</span></code> to find the first occurrence’s position and <code class="docutils literal notranslate"><span class="pre">mode=&quot;last&quot;</span></code> to find the last</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[35]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[36]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;First occurrence&quot;</span><span class="p">,</span> <span class="n">model</span><span class="o">.</span><span class="n">get_token_position</span><span class="p">(</span>
@@ -1631,7 +1652,7 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </div>
 <p>In general, tokenization is a pain, and full of gotchas. I highly recommend just playing around with different inputs and their tokenization and getting a feel for it. As another “fun” example, let’s look at the tokenization of arithmetic expressions - tokens do <em>not</em> contain consistent numbers of digits. (This makes it even more impressive that GPT-3 can do arithmetic!)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[36]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[37]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">to_str_tokens</span><span class="p">(</span><span class="s2">&quot;2342+2017=21445&quot;</span><span class="p">))</span>
@@ -1657,7 +1678,7 @@ <h3>Gotcha: <code class="docutils literal notranslate"><span class="pre">prepend
 End of Sequence (EOS), Beginning of Sequence (BOS) and Padding (PAD) tokens are all the same, <code class="docutils literal notranslate"><span class="pre">&lt;|endoftext|&gt;</span></code> with index <code class="docutils literal notranslate"><span class="pre">50256</span></code>.</p>
 <p><strong>Gotcha:</strong> You only want to prepend a BOS token at the <em>start</em> of a prompt. If you, eg, want to input a question followed by an answer, and want to tokenize these separately, you do <em>not</em> want to prepend_bos on the answer.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[37]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[38]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Logits shape by default (with BOS)&quot;</span><span class="p">,</span> <span class="n">model</span><span class="p">(</span><span class="s2">&quot;Hello World&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
@@ -1682,20 +1703,12 @@ <h3>Gotcha: <code class="docutils literal notranslate"><span class="pre">prepend
 Logits shape with BOS torch.Size([1, 3, 50257])
 </pre></div></div>
 </div>
-<div class="nboutput docutils container">
-<div class="prompt empty docutils container">
-</div>
-<div class="output_area docutils container">
-<div class="highlight"><pre>
-Logits shape without BOS - only 2 positions!
-</pre></div></div>
-</div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
- torch.Size([1, 2, 50257])
+Logits shape without BOS - only 2 positions! torch.Size([1, 2, 50257])
 </pre></div></div>
 </div>
 <p><code class="docutils literal notranslate"><span class="pre">prepend_bos</span></code> is a bit of a hack, and I’ve gone back and forth on what the correct default here is. The reason I do this is that transformers tend to treat the first token weirdly - this doesn’t really matter in training (where all inputs are &gt;1000 tokens), but this can be a big issue when investigating short prompts! The reason for this is that attention patterns are a probability distribution and so need to add up to one, so to simulate being “off” they normally look at the first token.
@@ -1704,7 +1717,7 @@ <h3>Gotcha: <code class="docutils literal notranslate"><span class="pre">prepend
 <p>(However, if you want to change the default behaviour to <em>not</em> prepending a BOS token, pass <code class="docutils literal notranslate"><span class="pre">default_prepend_bos=False</span></code> when you instantiate the model, e.g., <code class="docutils literal notranslate"><span class="pre">model</span> <span class="pre">=</span> <span class="pre">HookedTransformer.from_pretrained('gpt2',</span> <span class="pre">default_prepend_bos=False)</span></code>.)</p>
 <p>For example, the model can get much worse at Indirect Object Identification without a BOS (and with a name as the first token):</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[38]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[39]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">ioi_logits_with_bos</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="s2">&quot;Claire and Mary went to the shops, then Mary gave a bottle of milk to&quot;</span><span class="p">,</span> <span class="n">prepend_bos</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
@@ -1727,17 +1740,25 @@ <h3>Gotcha: <code class="docutils literal notranslate"><span class="pre">prepend
 Logit difference with BOS: 6.754
 </pre></div></div>
 </div>
-<div class="nboutput nblast docutils container">
+<div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
 Logit difference without BOS: 2.782
+</pre></div></div>
+</div>
+<div class="nboutput nblast docutils container">
+<div class="prompt empty docutils container">
+</div>
+<div class="output_area docutils container">
+<div class="highlight"><pre>
+
 </pre></div></div>
 </div>
 <p>Though, note that this also illustrates another gotcha - when <code class="docutils literal notranslate"><span class="pre">Claire</span></code> is at the start of a sentence (no preceding space), it’s actually <em>two</em> tokens, not one, which probably confuses the relevant circuit. (Note - in this test we put <code class="docutils literal notranslate"><span class="pre">prepend_bos=False</span></code>, because we want to analyse the tokenization of a specific string, not to give an input to the model!)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[39]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[40]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;| Claire| -&gt; </span><span class="si">{</span><span class="n">model</span><span class="o">.</span><span class="n">to_str_tokens</span><span class="p">(</span><span class="s1">&#39; Claire&#39;</span><span class="p">,</span><span class="w"> </span><span class="n">prepend_bos</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
@@ -1769,7 +1790,7 @@ <h2>Factored Matrix Class<a class="headerlink" href="#Factored-Matrix-Class" tit
 <h3>Basic Examples<a class="headerlink" href="#Basic-Examples" title="Permalink to this heading">#</a></h3>
 <p>We can use the basic class directly - let’s make a factored matrix directly and look at the basic operations:</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[40]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[41]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">IN_GITHUB</span><span class="p">:</span>
@@ -1800,7 +1821,7 @@ <h3>Basic Examples<a class="headerlink" href="#Basic-Examples" title="Permalink
 </div>
 <p>We can also look at the eigenvalues and singular values of the matrix. Note that, because the matrix is rank 2 but 5 by 5, the final 3 eigenvalues and singular values are zero - the factored class omits the zeros.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[41]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[42]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># NBVAL_IGNORE_OUTPUT</span>
@@ -1831,7 +1852,7 @@ <h3>Basic Examples<a class="headerlink" href="#Basic-Examples" title="Permalink
 </div>
 <p>We can multiply with other matrices - it automatically chooses the smallest possible dimension to factor along (here it’s 2, rather than 5)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[42]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[43]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="n">IN_GITHUB</span><span class="p">:</span>
@@ -1859,7 +1880,7 @@ <h3>Basic Examples<a class="headerlink" href="#Basic-Examples" title="Permalink
 </div>
 <p>If we want to collapse this back to an unfactored matrix, we can use the AB property to get the product:</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[43]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[44]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">AB_unfactored</span> <span class="o">=</span> <span class="n">AB_factor</span><span class="o">.</span><span class="n">AB</span>
@@ -1887,7 +1908,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <p>Let’s use FactoredMatrix to compute this for every head in the model! We use the helper <code class="docutils literal notranslate"><span class="pre">model.OV</span></code> to get the concatenated OV circuits for all heads across all layers in the model. This has the shape <code class="docutils literal notranslate"><span class="pre">[n_layers,</span> <span class="pre">n_heads,</span> <span class="pre">d_model,</span> <span class="pre">d_model]</span></code>, where <code class="docutils literal notranslate"><span class="pre">n_layers</span></code> and <code class="docutils literal notranslate"><span class="pre">n_heads</span></code> are batch dimensions and the final two dimensions are factorised as <code class="docutils literal notranslate"><span class="pre">[n_layers,</span> <span class="pre">n_heads,</span> <span class="pre">d_model,</span> <span class="pre">d_head]</span></code> and <code class="docutils literal notranslate"><span class="pre">[n_layers,</span> <span class="pre">n_heads,</span> <span class="pre">d_head,</span> <span class="pre">d_model]</span></code> matrices.</p>
 <p>We can then get the eigenvalues for this, where there are separate eigenvalues for each element of the batch (a <code class="docutils literal notranslate"><span class="pre">[n_layers,</span> <span class="pre">n_heads,</span> <span class="pre">d_head]</span></code> tensor of complex numbers), and calculate the copying score.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[44]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[45]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">OV_circuit_all_heads</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">OV</span>
@@ -1904,7 +1925,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </pre></div></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[45]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[46]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">OV_circuit_all_heads_eigenvalues</span> <span class="o">=</span> <span class="n">OV_circuit_all_heads</span><span class="o">.</span><span class="n">eigenvalues</span>
@@ -1923,7 +1944,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </pre></div></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[46]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[47]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">OV_copying_score</span> <span class="o">=</span> <span class="n">OV_circuit_all_heads_eigenvalues</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">real</span> <span class="o">/</span> <span class="n">OV_circuit_all_heads_eigenvalues</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
@@ -1939,9 +1960,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="29f8dfbd-47fb-4149-ba30-739032475d3b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("29f8dfbd-47fb-4149-ba30-739032475d3b")) {                    Plotly.newPlot(                        "29f8dfbd-47fb-4149-ba30-739032475d3b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="458e8707-ccf3-4567-a8b0-f931c572b246" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("458e8707-ccf3-4567-a8b0-f931c572b246")) {                    Plotly.newPlot(                        "458e8707-ccf3-4567-a8b0-f931c572b246",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('29f8dfbd-47fb-4149-ba30-739032475d3b');
+var gd = document.getElementById('458e8707-ccf3-4567-a8b0-f931c572b246');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1969,7 +1990,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </div>
 <p>Head 11 in Layer 11 (L11H11) has a high copying score, and if we plot the eigenvalues they look approximately as expected.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[47]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[48]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="o">=</span><span class="n">OV_circuit_all_heads_eigenvalues</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:]</span><span class="o">.</span><span class="n">real</span><span class="p">,</span> <span class="n">y</span><span class="o">=</span><span class="n">OV_circuit_all_heads_eigenvalues</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="p">:]</span><span class="o">.</span><span class="n">imag</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">&quot;Eigenvalues of Head L11H11 of GPT-2 Small&quot;</span><span class="p">,</span> <span class="n">xaxis</span><span class="o">=</span><span class="s2">&quot;Real&quot;</span><span class="p">,</span> <span class="n">yaxis</span><span class="o">=</span><span class="s2">&quot;Imaginary&quot;</span><span class="p">)</span>
@@ -1984,9 +2005,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="de463d3c-abc6-4c73-976f-c8458ad4e0e7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("de463d3c-abc6-4c73-976f-c8458ad4e0e7")) {                    Plotly.newPlot(                        "de463d3c-abc6-4c73-976f-c8458ad4e0e7",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="79eef653-2ecc-4731-aedd-e4b748085838" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("79eef653-2ecc-4731-aedd-e4b748085838")) {                    Plotly.newPlot(                        "79eef653-2ecc-4731-aedd-e4b748085838",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('de463d3c-abc6-4c73-976f-c8458ad4e0e7');
+var gd = document.getElementById('79eef653-2ecc-4731-aedd-e4b748085838');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2014,7 +2035,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </div>
 <p>We can even look at the full OV circuit, from the input tokens to output tokens: <span class="math notranslate nohighlight">\(W_E W_V W_O W_U\)</span>. This is a <code class="docutils literal notranslate"><span class="pre">[d_vocab,</span> <span class="pre">d_vocab]==[50257,</span> <span class="pre">50257]</span></code> matrix, so absolutely enormous, even for a single head. But with the FactoredMatrix class, we can compute the full eigenvalue copying score of every head in a few seconds.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[48]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[49]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">full_OV_circuit</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">embed</span><span class="o">.</span><span class="n">W_E</span> <span class="o">@</span> <span class="n">OV_circuit_all_heads</span> <span class="o">@</span> <span class="n">model</span><span class="o">.</span><span class="n">unembed</span><span class="o">.</span><span class="n">W_U</span>
@@ -2031,7 +2052,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </pre></div></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[49]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[50]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">full_OV_circuit_eigenvalues</span> <span class="o">=</span> <span class="n">full_OV_circuit</span><span class="o">.</span><span class="n">eigenvalues</span>
@@ -2050,7 +2071,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </pre></div></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[50]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[51]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">full_OV_copying_score</span> <span class="o">=</span> <span class="n">full_OV_circuit_eigenvalues</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">real</span> <span class="o">/</span> <span class="n">full_OV_circuit_eigenvalues</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
@@ -2066,9 +2087,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e218ffb2-72ea-46fa-b8e3-fe441e03c97b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e218ffb2-72ea-46fa-b8e3-fe441e03c97b")) {                    Plotly.newPlot(                        "e218ffb2-72ea-46fa-b8e3-fe441e03c97b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b62ac579-268a-4cb8-a4db-de9c144e5855" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b62ac579-268a-4cb8-a4db-de9c144e5855")) {                    Plotly.newPlot(                        "b62ac579-268a-4cb8-a4db-de9c144e5855",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e218ffb2-72ea-46fa-b8e3-fe441e03c97b');
+var gd = document.getElementById('b62ac579-268a-4cb8-a4db-de9c144e5855');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2096,7 +2117,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </div>
 <p>Interestingly, these are highly (but not perfectly!) correlated. I’m not sure what to read from this, or what’s up with the weird outlier heads!</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[51]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[52]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="o">=</span><span class="n">full_OV_copying_score</span><span class="o">.</span><span class="n">flatten</span><span class="p">(),</span> <span class="n">y</span><span class="o">=</span><span class="n">OV_copying_score</span><span class="o">.</span><span class="n">flatten</span><span class="p">(),</span> <span class="n">hover_name</span><span class="o">=</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;L</span><span class="si">{</span><span class="n">layer</span><span class="si">}</span><span class="s2">H</span><span class="si">{</span><span class="n">head</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">layer</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">12</span><span class="p">)</span> <span class="k">for</span> <span class="n">head</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">12</span><span class="p">)],</span> <span class="n">title</span><span class="o">=</span><span class="s2">&quot;OV Copying Score for each head in GPT-2 Small&quot;</span><span class="p">,</span> <span class="n">xaxis</span><span class="o">=</span><span class="s2">&quot;Full OV Copying Score&quot;</span><span class="p">,</span> <span class="n">yaxis</span><span class="o">=</span><span class="s2">&quot;OV Copying Score&quot;</span><span class="p">)</span>
@@ -2111,9 +2132,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5")) {                    Plotly.newPlot(                        "dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="3c8c77e9-6054-4886-85c5-8868e8e7c01c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("3c8c77e9-6054-4886-85c5-8868e8e7c01c")) {                    Plotly.newPlot(                        "3c8c77e9-6054-4886-85c5-8868e8e7c01c",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5');
+var gd = document.getElementById('3c8c77e9-6054-4886-85c5-8868e8e7c01c');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2140,7 +2161,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 </html></div>
 </div>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[52]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[53]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Token 256 - the most common pair of ASCII characters: |</span><span class="si">{</span><span class="n">model</span><span class="o">.</span><span class="n">to_string</span><span class="p">(</span><span class="mi">256</span><span class="p">)</span><span class="si">}</span><span class="s2">|&quot;</span><span class="p">)</span>
@@ -2166,7 +2187,7 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <h2>Generating Text<a class="headerlink" href="#Generating-Text" title="Permalink to this heading">#</a></h2>
 <p>TransformerLens also has basic text generation functionality, which can be useful for generally exploring what the model is capable of (thanks to Ansh Radhakrishnan for adding this!). This is pretty rough functionality, and where possible I recommend using more established libraries like HuggingFace for this.</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[53]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[54]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># NBVAL_IGNORE_OUTPUT</span>
@@ -2178,10 +2199,10 @@ <h2>Generating Text<a class="headerlink" href="#Generating-Text" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ca0421e9c3074fe38697b4e54fe160b3"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "ef67630185584553867488aa7abe9fa9"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[53]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[54]:
 </pre></div>
 </div>
 <div class="output_area docutils container">
@@ -2203,7 +2224,7 @@ <h3>Toy Example<a class="headerlink" href="#Toy-Example" title="Permalink to thi
 <p>We define a basic network with two layers that each take a scalar input <span class="math notranslate nohighlight">\(x\)</span>, square it, and add a constant: <span class="math notranslate nohighlight">\(x_0=x\)</span>, <span class="math notranslate nohighlight">\(x_1=x_0^2+3\)</span>, <span class="math notranslate nohighlight">\(x_2=x_1^2-4\)</span>.</p>
 <p>We wrap the input, each layer’s output, and the intermediate value of each layer (the square) in a hook point.</p>
 <div class="nbinput nblast docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[54]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[55]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><br/><span></span><span class="kn">from</span> <span class="nn">transformer_lens.hook_points</span> <span class="kn">import</span> <span class="n">HookedRootModule</span><span class="p">,</span> <span class="n">HookPoint</span>
@@ -2251,7 +2272,7 @@ <h3>Toy Example<a class="headerlink" href="#Toy-Example" title="Permalink to thi
 <p>We can add a cache, to save the activation at each hook point</p>
 <p>(There’s a custom <code class="docutils literal notranslate"><span class="pre">run_with_cache</span></code> function on the root module as a convenience, which is a wrapper around model.forward that return model_out, cache_object - we could also manually add hooks with <code class="docutils literal notranslate"><span class="pre">run_with_hooks</span></code> that store activations in a global caching dictionary. This is often useful if we only want to store, eg, subsets or functions of some activations.)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[55]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[56]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><br/><span></span><span class="n">out</span><span class="p">,</span> <span class="n">cache</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">run_with_cache</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">5.0</span><span class="p">))</span>
@@ -2276,7 +2297,7 @@ <h3>Toy Example<a class="headerlink" href="#Toy-Example" title="Permalink to thi
 </div>
 <p>We can also use hooks to intervene on activations - eg, we can set the intermediate value in layer 2 to zero to change the output to -5</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[56]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[57]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><br/><span></span><span class="k">def</span> <span class="nf">set_to_zero_hook</span><span class="p">(</span><span class="n">tensor</span><span class="p">,</span> <span class="n">hook</span><span class="p">):</span>
@@ -2315,7 +2336,7 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 of label was used.</p>
 <p>Here are graphs of the schedules for several checkpointed models: (note that the first 3 use a log scale, latter 2 use a linear scale)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[57]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[58]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">transformer_lens.loading_from_pretrained</span> <span class="kn">import</span> <span class="n">get_checkpoint_labels</span>
@@ -2336,9 +2357,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="f75ed1fa-2d37-438f-a502-ed44a36bd3db" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f75ed1fa-2d37-438f-a502-ed44a36bd3db")) {                    Plotly.newPlot(                        "f75ed1fa-2d37-438f-a502-ed44a36bd3db",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="76053182-4b0b-44d5-9e6f-dc68cfcabb60" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("76053182-4b0b-44d5-9e6f-dc68cfcabb60")) {                    Plotly.newPlot(                        "76053182-4b0b-44d5-9e6f-dc68cfcabb60",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f75ed1fa-2d37-438f-a502-ed44a36bd3db');
+var gd = document.getElementById('76053182-4b0b-44d5-9e6f-dc68cfcabb60');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2372,9 +2393,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="3bb32d65-c865-49b8-9cc7-92f6460f7bf6" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("3bb32d65-c865-49b8-9cc7-92f6460f7bf6")) {                    Plotly.newPlot(                        "3bb32d65-c865-49b8-9cc7-92f6460f7bf6",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="22829199-cf37-49cf-8817-a42bc6f79448" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("22829199-cf37-49cf-8817-a42bc6f79448")) {                    Plotly.newPlot(                        "22829199-cf37-49cf-8817-a42bc6f79448",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('3bb32d65-c865-49b8-9cc7-92f6460f7bf6');
+var gd = document.getElementById('22829199-cf37-49cf-8817-a42bc6f79448');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2408,9 +2429,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="37192a3b-da9b-4923-8d68-311c63c28642" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("37192a3b-da9b-4923-8d68-311c63c28642")) {                    Plotly.newPlot(                        "37192a3b-da9b-4923-8d68-311c63c28642",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="86fd24b0-a205-4e6c-8105-f79fcf57c539" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("86fd24b0-a205-4e6c-8105-f79fcf57c539")) {                    Plotly.newPlot(                        "86fd24b0-a205-4e6c-8105-f79fcf57c539",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('37192a3b-da9b-4923-8d68-311c63c28642');
+var gd = document.getElementById('86fd24b0-a205-4e6c-8105-f79fcf57c539');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2444,9 +2465,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="d996cbd1-4f6e-452c-8efb-5d303fe6d5ab" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d996cbd1-4f6e-452c-8efb-5d303fe6d5ab")) {                    Plotly.newPlot(                        "d996cbd1-4f6e-452c-8efb-5d303fe6d5ab",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="4a6e81dc-ddca-460f-873f-0a213188ab89" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4a6e81dc-ddca-460f-873f-0a213188ab89")) {                    Plotly.newPlot(                        "4a6e81dc-ddca-460f-873f-0a213188ab89",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d996cbd1-4f6e-452c-8efb-5d303fe6d5ab');
+var gd = document.getElementById('4a6e81dc-ddca-460f-873f-0a213188ab89');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2480,9 +2501,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="28df3ce6-a5fb-4e23-a85c-7ef4a03ed832" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("28df3ce6-a5fb-4e23-a85c-7ef4a03ed832")) {                    Plotly.newPlot(                        "28df3ce6-a5fb-4e23-a85c-7ef4a03ed832",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e2877398-410c-47b5-9f61-1fd97414d652" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e2877398-410c-47b5-9f61-1fd97414d652")) {                    Plotly.newPlot(                        "e2877398-410c-47b5-9f61-1fd97414d652",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('28df3ce6-a5fb-4e23-a85c-7ef4a03ed832');
+var gd = document.getElementById('e2877398-410c-47b5-9f61-1fd97414d652');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2516,7 +2537,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 default just runs it on 4 sequences of 384 tokens repeated twice. These results totally don’t do the paper justice - go check it out if you want to see the full results!)</p>
 <p>In the interests of time and memory, let’s look at a handful of checkpoints (chosen to be around the phase change), indices <code class="docutils literal notranslate"><span class="pre">[10,</span> <span class="pre">25,</span> <span class="pre">35,</span> <span class="pre">60,</span> <span class="pre">-1]</span></code>. These are roughly 22M, 200M, 500M, 1.6B and 21.8B tokens through training, respectively. (I generally recommend looking things up based on indices, rather than checkpoint value!).</p>
 <div class="nbinput nblast docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[58]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[59]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">transformer_lens</span> <span class="kn">import</span> <span class="n">evals</span>
@@ -2532,7 +2553,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 </div>
 <p>We load the models, cache them in a list, and</p>
 <div class="nbinput nblast docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[59]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[60]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">if</span> <span class="ow">not</span> <span class="n">IN_GITHUB</span><span class="p">:</span>
@@ -2552,7 +2573,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 <p>We can plot this, and see there’s a sharp shift from ~200-500M tokens trained on (note the log scale on the x axis). Interestingly, this is notably earlier than the phase transition in the paper, I’m not sure what’s up with that.</p>
 <p>(To contextualise the numbers, the tokens in the random sequence are uniformly chosen from the first 20,000 tokens (out of ~48,000 total), so random performance is at least <span class="math notranslate nohighlight">\(\ln(20000)\approx 10\)</span>. A naive strategy like “randomly choose a token that’s already appeared in the first half of the sequence (384 elements)” would get <span class="math notranslate nohighlight">\(\ln(384)\approx 5.95\)</span>, so the model is doing pretty well here.)</p>
 <div class="nbinput docutils container">
-<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[60]:
+<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[61]:
 </pre></div>
 </div>
 <div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">line</span><span class="p">(</span><span class="n">induction_losses</span><span class="p">,</span> <span class="n">x</span><span class="o">=</span><span class="n">tokens_trained_on</span><span class="p">,</span> <span class="n">xaxis</span><span class="o">=</span><span class="s2">&quot;Tokens Trained On&quot;</span><span class="p">,</span> <span class="n">yaxis</span><span class="o">=</span><span class="s2">&quot;Induction Loss&quot;</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">&quot;Induction Loss over training: solu-2l&quot;</span><span class="p">,</span> <span class="n">markers</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">log_x</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
@@ -2567,9 +2588,9 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="0ebddc76-ece2-4920-a2c7-60871b11afed" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("0ebddc76-ece2-4920-a2c7-60871b11afed")) {                    Plotly.newPlot(                        "0ebddc76-ece2-4920-a2c7-60871b11afed",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e8c48c58-6618-4eb6-8790-24b50ffe4409" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e8c48c58-6618-4eb6-8790-24b50ffe4409")) {                    Plotly.newPlot(                        "e8c48c58-6618-4eb6-8790-24b50ffe4409",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('0ebddc76-ece2-4920-a2c7-60871b11afed');
+var gd = document.getElementById('e8c48c58-6618-4eb6-8790-24b50ffe4409');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2596,7 +2617,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 </html></div>
 </div>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"d027f2bb85564a00bf60cfe08b9b71ff": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "14879e61652b40c5ba52609fd5646866": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "1c96e44af3b24a34aaff4045e496a27d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d027f2bb85564a00bf60cfe08b9b71ff", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_14879e61652b40c5ba52609fd5646866", "tabbable": null, "tooltip": null, "value": 12.0}}, "8aa3d272612246cb9ffad05643ac4d7d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "537c952c420e4ae1aea0035c609ce165": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "181e4adfc3bc45cd965a56602246337a": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8aa3d272612246cb9ffad05643ac4d7d", "placeholder": "\u200b", "style": "IPY_MODEL_537c952c420e4ae1aea0035c609ce165", "tabbable": null, "tooltip": null, "value": "100%"}}, "0a116157d07c4ab1801aae54d0153bfd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e75a1f080c744b1c8939f0abb8e708fc": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "27310f32e9974dcd94aa2b9c49c1e6e7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0a116157d07c4ab1801aae54d0153bfd", "placeholder": "\u200b", "style": "IPY_MODEL_e75a1f080c744b1c8939f0abb8e708fc", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:19&lt;00:00,\u2007\u20071.64s/it]"}}, "6c874564389b4cc08af3017cb4823571": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8886699251a949f49e757b9fdc7ca890": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_181e4adfc3bc45cd965a56602246337a", "IPY_MODEL_1c96e44af3b24a34aaff4045e496a27d", "IPY_MODEL_27310f32e9974dcd94aa2b9c49c1e6e7"], "layout": "IPY_MODEL_6c874564389b4cc08af3017cb4823571", "tabbable": null, "tooltip": null}}, "2ce16701f86e465fa67e824fed52d560": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "fb7967775a7c426ab326120208b1c1d0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "ed792737675641488478dfe9e706af53": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2ce16701f86e465fa67e824fed52d560", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_fb7967775a7c426ab326120208b1c1d0", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "ec306a831c0c4f55b9058110a79d8978": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "83789bfd75804ec290640f71987c3492": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "df6f557d81e94e78b53b3b8791c846c4": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ec306a831c0c4f55b9058110a79d8978", "placeholder": "\u200b", "style": "IPY_MODEL_83789bfd75804ec290640f71987c3492", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "1f2d34420a9d422db4c26669c9eece1b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e3d82c176b4b452e93686bf9f1c9d3e1": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e8077a539c2f4fe7a0bc96e4cd70fa7e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_1f2d34420a9d422db4c26669c9eece1b", "placeholder": "\u200b", "style": "IPY_MODEL_e3d82c176b4b452e93686bf9f1c9d3e1", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:01&lt;00:00,\u2007248MB/s]"}}, "9d118b82362542f59e0227f8f93cf219": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "504b41851a444c31b688e5e8b063add5": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_df6f557d81e94e78b53b3b8791c846c4", "IPY_MODEL_ed792737675641488478dfe9e706af53", "IPY_MODEL_e8077a539c2f4fe7a0bc96e4cd70fa7e"], "layout": "IPY_MODEL_9d118b82362542f59e0227f8f93cf219", "tabbable": null, "tooltip": null}}, "d23203daf6b546699057c5635a148bad": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3184aa14ca4b42eca4a28d1cdd508bea": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "397d196b08ea47da94f5d96dd6276340": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d23203daf6b546699057c5635a148bad", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_3184aa14ca4b42eca4a28d1cdd508bea", "tabbable": null, "tooltip": null, "value": 124.0}}, "34b16debba6742959118694ecadd4e80": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b224d7466bda47f88fa7756be3f08ef5": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "5ef4034cdc7c4b43a254cd124d41d7ab": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_34b16debba6742959118694ecadd4e80", "placeholder": "\u200b", "style": "IPY_MODEL_b224d7466bda47f88fa7756be3f08ef5", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "db618c1d50364bed9940e58ee948cedf": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5054bc9f6dc94e5a938d3b2a410a235f": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c8831f07d5264cae9518fd997bd1464f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_db618c1d50364bed9940e58ee948cedf", "placeholder": "\u200b", "style": "IPY_MODEL_5054bc9f6dc94e5a938d3b2a410a235f", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200722.2kB/s]"}}, "c277db05726a40f691e0fec1426c30e2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8efd0c3ea3bc4e5d8c1b966bff063c3a": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_5ef4034cdc7c4b43a254cd124d41d7ab", "IPY_MODEL_397d196b08ea47da94f5d96dd6276340", "IPY_MODEL_c8831f07d5264cae9518fd997bd1464f"], "layout": "IPY_MODEL_c277db05726a40f691e0fec1426c30e2", "tabbable": null, "tooltip": null}}, "40f00596573845cfb4f6ee3bf57fcf62": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1e61511b645b4c07aabc82cde43f0707": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "fee2c6fbc51f44eab84f3b42742d91e1": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_40f00596573845cfb4f6ee3bf57fcf62", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_1e61511b645b4c07aabc82cde43f0707", "tabbable": null, "tooltip": null, "value": 26.0}}, "f4944bf0e2044a9bb30a6e728cf9a4b0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "fcee2cda87b34b6daaaa18c4471f41f0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "75e910c8d71048ac9a0431c21969d577": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f4944bf0e2044a9bb30a6e728cf9a4b0", "placeholder": "\u200b", "style": "IPY_MODEL_fcee2cda87b34b6daaaa18c4471f41f0", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "fcc3cac4665349559e159dd7b1be5c7f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "35de1507ddfa4b87b3e78c74855788cd": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1510c65a28504ad5b352dcd079536344": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_fcc3cac4665349559e159dd7b1be5c7f", "placeholder": "\u200b", "style": "IPY_MODEL_35de1507ddfa4b87b3e78c74855788cd", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.50kB/s]"}}, "f6a821330d1046798295f553b4e852c9": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5df370f098ea41fba5fb462b17adbe25": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_75e910c8d71048ac9a0431c21969d577", "IPY_MODEL_fee2c6fbc51f44eab84f3b42742d91e1", "IPY_MODEL_1510c65a28504ad5b352dcd079536344"], "layout": "IPY_MODEL_f6a821330d1046798295f553b4e852c9", "tabbable": null, "tooltip": null}}, "b5cfa8bab51f4768968002c31b9c4ccc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5168126e6e2640259859e571cf02ac2b": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "483d3536ea8644e0932ff376a08a7509": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b5cfa8bab51f4768968002c31b9c4ccc", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_5168126e6e2640259859e571cf02ac2b", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "12c5e2e95f084803834b82ff36d5a9aa": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b5b250cb0b404e5dbfb096747062a997": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "f88e08bf6447493e9ee596e52053d053": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_12c5e2e95f084803834b82ff36d5a9aa", "placeholder": "\u200b", "style": "IPY_MODEL_b5b250cb0b404e5dbfb096747062a997", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "b43f9e4d856841f4a148132df957a6bb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e36e177cd6b047a6a4ef203d26f44bb3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c4fc9d6b249f46218d1b6b1088d1852f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b43f9e4d856841f4a148132df957a6bb", "placeholder": "\u200b", "style": "IPY_MODEL_e36e177cd6b047a6a4ef203d26f44bb3", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200714.5MB/s]"}}, "d7e889b34a51418193c95f469bf95c6d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d5f0c8842a284fdda4dfa3206605c35f": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_f88e08bf6447493e9ee596e52053d053", "IPY_MODEL_483d3536ea8644e0932ff376a08a7509", "IPY_MODEL_c4fc9d6b249f46218d1b6b1088d1852f"], "layout": "IPY_MODEL_d7e889b34a51418193c95f469bf95c6d", "tabbable": null, "tooltip": null}}, "a82d2103fcf049959a85dad02d097310": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bae937ca0f53411b921ad90d6d430654": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "7164ecd232594c6d854b1365370e4223": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a82d2103fcf049959a85dad02d097310", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_bae937ca0f53411b921ad90d6d430654", "tabbable": null, "tooltip": null, "value": 456318.0}}, "39fcd78619f74e7fa34ba3c717ecde79": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8d0044c9d9594cdc99a744683335c85f": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "17bb4044cad04889b7a3185f7cca5d61": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_39fcd78619f74e7fa34ba3c717ecde79", "placeholder": "\u200b", "style": "IPY_MODEL_8d0044c9d9594cdc99a744683335c85f", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "9bd1760890dd4e298154efe56e5a757d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4eb29c030d0b4615a334a1bf3f16ded1": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "2eb8521bbd3d441180ebfef6c4e91184": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9bd1760890dd4e298154efe56e5a757d", "placeholder": "\u200b", "style": "IPY_MODEL_4eb29c030d0b4615a334a1bf3f16ded1", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u20072.48MB/s]"}}, "79d408b7c64b44d0922d0edcc376a946": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a678c766e93d424daa0150a52a8f38a2": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_17bb4044cad04889b7a3185f7cca5d61", "IPY_MODEL_7164ecd232594c6d854b1365370e4223", "IPY_MODEL_2eb8521bbd3d441180ebfef6c4e91184"], "layout": "IPY_MODEL_79d408b7c64b44d0922d0edcc376a946", "tabbable": null, "tooltip": null}}, "f38841b822954662b43dba4c39e860e5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9069a6ad426248c09b77ea95dd5aee4a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "68523653d5074bb6870277f003d5d605": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f38841b822954662b43dba4c39e860e5", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_9069a6ad426248c09b77ea95dd5aee4a", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "1fa06c806c4a4980bbe87f0746c9a335": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d8ec5600d09e4881a86bbf234f64eee7": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "af4c8704f5ac48e18155eafc53cb00cf": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_1fa06c806c4a4980bbe87f0746c9a335", "placeholder": "\u200b", "style": "IPY_MODEL_d8ec5600d09e4881a86bbf234f64eee7", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "7edd3fced4af4b77a0ddce7023ab8b7b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7efc08a62c6f40298d362be028692ce6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6e868c8f6c0448e7b61f10235b8c78a3": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7edd3fced4af4b77a0ddce7023ab8b7b", "placeholder": "\u200b", "style": "IPY_MODEL_7efc08a62c6f40298d362be028692ce6", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u200717.6MB/s]"}}, "7cb2aaf5bf784a5f952b312cc762a687": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "da7af55f6bb5479b852a41df346f73ee": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_af4c8704f5ac48e18155eafc53cb00cf", "IPY_MODEL_68523653d5074bb6870277f003d5d605", "IPY_MODEL_6e868c8f6c0448e7b61f10235b8c78a3"], "layout": "IPY_MODEL_7cb2aaf5bf784a5f952b312cc762a687", "tabbable": null, "tooltip": null}}, "d99e129cdb3147aa913c3c969c4c5910": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "81593b1d4ed44438ac88b2d7a8b8cb96": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "0813474f79c445e4969ff3384cba2b31": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d99e129cdb3147aa913c3c969c4c5910", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_81593b1d4ed44438ac88b2d7a8b8cb96", "tabbable": null, "tooltip": null, "value": 50.0}}, "a452f9720b0141f18f4f1585cfbcf102": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "051ba86a5dc04ae7a169b5a2f5d883db": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "099201fc3592463a99cefa3adf51b80e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a452f9720b0141f18f4f1585cfbcf102", "placeholder": "\u200b", "style": "IPY_MODEL_051ba86a5dc04ae7a169b5a2f5d883db", "tabbable": null, "tooltip": null, "value": "100%"}}, "718432154ef24d44aa6133126ac22094": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1ed9a2e74124405fb3abc69c91bd1ea6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "5adce76b5f7845b087083c2207ab56d3": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_718432154ef24d44aa6133126ac22094", "placeholder": "\u200b", "style": "IPY_MODEL_1ed9a2e74124405fb3abc69c91bd1ea6", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200722.98it/s]"}}, "6114d823a31a44f286d95a2e9ce2bf33": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ca0421e9c3074fe38697b4e54fe160b3": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_099201fc3592463a99cefa3adf51b80e", "IPY_MODEL_0813474f79c445e4969ff3384cba2b31", "IPY_MODEL_5adce76b5f7845b087083c2207ab56d3"], "layout": "IPY_MODEL_6114d823a31a44f286d95a2e9ce2bf33", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"2974465a897b4ac9ba7d573f8e1ea5f8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "58a2ce5740e543bdaa92d598b1b084dc": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "977babee536f42158a1d81b4b7aa208d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2974465a897b4ac9ba7d573f8e1ea5f8", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_58a2ce5740e543bdaa92d598b1b084dc", "tabbable": null, "tooltip": null, "value": 12.0}}, "e4da68381e90404db807b341d42c96a8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4f9bedf73f97497db82c1b58ff5117bf": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8480a1acfe1a4929becf5c1eeb2df5f1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e4da68381e90404db807b341d42c96a8", "placeholder": "\u200b", "style": "IPY_MODEL_4f9bedf73f97497db82c1b58ff5117bf", "tabbable": null, "tooltip": null, "value": "100%"}}, "22fa2830a0ab4ae4bb190db8ab08f59c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bd751bbba24c4154acc75b88e0f3a87a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "fdf5d6542de342d08a2d9a0482eda1d7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_22fa2830a0ab4ae4bb190db8ab08f59c", "placeholder": "\u200b", "style": "IPY_MODEL_bd751bbba24c4154acc75b88e0f3a87a", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:19&lt;00:00,\u2007\u20071.65s/it]"}}, "76995f1052794da19381d92e27c1de96": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ddd91d21a45e49a8a90fd6693fc0fe90": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_8480a1acfe1a4929becf5c1eeb2df5f1", "IPY_MODEL_977babee536f42158a1d81b4b7aa208d", "IPY_MODEL_fdf5d6542de342d08a2d9a0482eda1d7"], "layout": "IPY_MODEL_76995f1052794da19381d92e27c1de96", "tabbable": null, "tooltip": null}}, "ce2ad39062644901b02faaefcf98b910": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "16df306b166340878aae8ee0d7d2af94": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4116100d429b4b9faf99f919b8cba2fd": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ce2ad39062644901b02faaefcf98b910", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_16df306b166340878aae8ee0d7d2af94", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "7f7ad53041c6499e9178b752e3f0bf66": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a3feec14402a46afa537597c59f70647": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "c70847ad8e5543748ffbab29446b51c1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7f7ad53041c6499e9178b752e3f0bf66", "placeholder": "\u200b", "style": "IPY_MODEL_a3feec14402a46afa537597c59f70647", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "aa07cd1a18be4feda9dfaae9577809b5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5fbc42bb13f64054a6dbbd8c58ecfc59": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6304f591778f49058f9403f77c821c56": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_aa07cd1a18be4feda9dfaae9577809b5", "placeholder": "\u200b", "style": "IPY_MODEL_5fbc42bb13f64054a6dbbd8c58ecfc59", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:01&lt;00:00,\u2007198MB/s]"}}, "1c150b40a42e403b848a74fd683a4788": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2b653bed330e47bfaf1eb97604ffb148": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_c70847ad8e5543748ffbab29446b51c1", "IPY_MODEL_4116100d429b4b9faf99f919b8cba2fd", "IPY_MODEL_6304f591778f49058f9403f77c821c56"], "layout": "IPY_MODEL_1c150b40a42e403b848a74fd683a4788", "tabbable": null, "tooltip": null}}, "6e974e0f6e104aee841709766cbfe512": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4025e90abd624e92ad601372fe03f72a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "45071b25ff23496094a139022c8edc54": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6e974e0f6e104aee841709766cbfe512", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4025e90abd624e92ad601372fe03f72a", "tabbable": null, "tooltip": null, "value": 124.0}}, "19da29c1b3cf4fae9710cd70f0baf271": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "30e033f4a4a745f99416fa6c47e6b1b8": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "3a73a2ce3d684e57af5ead4d970bb546": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_19da29c1b3cf4fae9710cd70f0baf271", "placeholder": "\u200b", "style": "IPY_MODEL_30e033f4a4a745f99416fa6c47e6b1b8", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "3adaa7786f684ff6abb501929b91bfeb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "236a3fe34dbb46b9a945a6c32dc4ed37": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0093c35dd59c47adaed249b404d1a0b1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3adaa7786f684ff6abb501929b91bfeb", "placeholder": "\u200b", "style": "IPY_MODEL_236a3fe34dbb46b9a945a6c32dc4ed37", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.8kB/s]"}}, "2fbd1d8558174aaeae9046da37a2dc08": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "2f95bda34e124bdb8ad42b97968c2560": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_3a73a2ce3d684e57af5ead4d970bb546", "IPY_MODEL_45071b25ff23496094a139022c8edc54", "IPY_MODEL_0093c35dd59c47adaed249b404d1a0b1"], "layout": "IPY_MODEL_2fbd1d8558174aaeae9046da37a2dc08", "tabbable": null, "tooltip": null}}, "3383a9989d6a4375bccecc436f7df6c0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a05caff4cdcb42e8813ccaf3072ea092": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f50d3b1ce11a44d8b5801bdd32a86fa8": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3383a9989d6a4375bccecc436f7df6c0", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_a05caff4cdcb42e8813ccaf3072ea092", "tabbable": null, "tooltip": null, "value": 26.0}}, "f825f61e07764da39007e09a988d2df0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "dee4a33c12ba4787ad99f56355b782af": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d671580b566f41a3b09b9e31d0a97129": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f825f61e07764da39007e09a988d2df0", "placeholder": "\u200b", "style": "IPY_MODEL_dee4a33c12ba4787ad99f56355b782af", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "a95e7f64a7d240529cacf7a5a9e1c06b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "40bb01d6e86d499a80c048f392bcbe29": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "a3a46ce06ec04834810c4ab96c959684": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a95e7f64a7d240529cacf7a5a9e1c06b", "placeholder": "\u200b", "style": "IPY_MODEL_40bb01d6e86d499a80c048f392bcbe29", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20075.03kB/s]"}}, "7129d439a9d549d48802b08586b8c24f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1bfe4dc8f9d64ed3a70a0e6e41b61d34": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d671580b566f41a3b09b9e31d0a97129", "IPY_MODEL_f50d3b1ce11a44d8b5801bdd32a86fa8", "IPY_MODEL_a3a46ce06ec04834810c4ab96c959684"], "layout": "IPY_MODEL_7129d439a9d549d48802b08586b8c24f", "tabbable": null, "tooltip": null}}, "25f38d8b724d481397c21ffd4e2ae408": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "546e22db36d04372a0aa187adbcf0262": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "975c4858e139485f8f0aa84771e564a4": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_25f38d8b724d481397c21ffd4e2ae408", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_546e22db36d04372a0aa187adbcf0262", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "bc420542d1d34e14978dcc6d9d0471e3": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c741d615fd45485b899d67710062ed85": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "98db25d1775c4574b9702ae9d27f9d76": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_bc420542d1d34e14978dcc6d9d0471e3", "placeholder": "\u200b", "style": "IPY_MODEL_c741d615fd45485b899d67710062ed85", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "355529cac414494c83e338e54c92a4cc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "72e1c6506989403db938acb8a05caa9a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "3b38cd8e7a4749a49d9442a541dd6d69": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_355529cac414494c83e338e54c92a4cc", "placeholder": "\u200b", "style": "IPY_MODEL_72e1c6506989403db938acb8a05caa9a", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200717.5MB/s]"}}, "a6d329232ced49958c43f07f02862115": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "363a53f62bad40f38d9b57415fcdd27a": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_98db25d1775c4574b9702ae9d27f9d76", "IPY_MODEL_975c4858e139485f8f0aa84771e564a4", "IPY_MODEL_3b38cd8e7a4749a49d9442a541dd6d69"], "layout": "IPY_MODEL_a6d329232ced49958c43f07f02862115", "tabbable": null, "tooltip": null}}, "b49e96fd18bc45e981e58cefbd348841": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4045d70e18ef41f892b496c2457200e8": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "4ec86ba49fc04f1ea410f2a37bf7ffae": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b49e96fd18bc45e981e58cefbd348841", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4045d70e18ef41f892b496c2457200e8", "tabbable": null, "tooltip": null, "value": 456318.0}}, "46833033a8eb479ea4c5c87af7d997bb": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5ad68d8e26e44e76a786ef4b57a2263d": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "aec5d70e86d84aef8fe50515973443a8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_46833033a8eb479ea4c5c87af7d997bb", "placeholder": "\u200b", "style": "IPY_MODEL_5ad68d8e26e44e76a786ef4b57a2263d", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "c6ee4432596b4ff999fe1bddd0d46240": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "697afa5733c6440f907ee2c1ffb2584e": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "4d16a5f6738f41918a9be1a5c6911d80": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c6ee4432596b4ff999fe1bddd0d46240", "placeholder": "\u200b", "style": "IPY_MODEL_697afa5733c6440f907ee2c1ffb2584e", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200735.6MB/s]"}}, "485b5446c8864360a4f063c6992e772a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "baa802cac1bf4a77bd2d6f2da4e0c206": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_aec5d70e86d84aef8fe50515973443a8", "IPY_MODEL_4ec86ba49fc04f1ea410f2a37bf7ffae", "IPY_MODEL_4d16a5f6738f41918a9be1a5c6911d80"], "layout": "IPY_MODEL_485b5446c8864360a4f063c6992e772a", "tabbable": null, "tooltip": null}}, "f5bf41cb59ed4db4b184f40d99bfbc1a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ab0021bee3b94de0965c25c38a7962f2": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "db49cd92e84f4f20b4d44fe5829790df": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f5bf41cb59ed4db4b184f40d99bfbc1a", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_ab0021bee3b94de0965c25c38a7962f2", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "9fce92e16f574800b2a9add4b6c10ef8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "bfdbeaa99bc942abb6a71e94629483ab": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0f1f32e4ccb44e77b4a1aab830f601a5": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9fce92e16f574800b2a9add4b6c10ef8", "placeholder": "\u200b", "style": "IPY_MODEL_bfdbeaa99bc942abb6a71e94629483ab", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "47b00b0a700a470cba9e0970646bf5a3": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "459712eff9964201b95e3cb31857e8da": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "70759237fca04d3084c2a5df2a30f271": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_47b00b0a700a470cba9e0970646bf5a3", "placeholder": "\u200b", "style": "IPY_MODEL_459712eff9964201b95e3cb31857e8da", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20079.38MB/s]"}}, "c666ac6d4533412e922fc4b9a1084fc1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "85239cb8d6ba4d288b001290e919d816": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_0f1f32e4ccb44e77b4a1aab830f601a5", "IPY_MODEL_db49cd92e84f4f20b4d44fe5829790df", "IPY_MODEL_70759237fca04d3084c2a5df2a30f271"], "layout": "IPY_MODEL_c666ac6d4533412e922fc4b9a1084fc1", "tabbable": null, "tooltip": null}}, "0347d987e2964dd4a31348434040c0c7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "858356c58e784bc4b5778ed67747a0d1": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "69da672452a94ea68b438b3be68f1a24": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0347d987e2964dd4a31348434040c0c7", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_858356c58e784bc4b5778ed67747a0d1", "tabbable": null, "tooltip": null, "value": 50.0}}, "33b3f2ed76434c13b68809069903b693": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ae9c1384a8ef40a9b2a32844f1646716": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1dfb46a04fe14273b07f0621a7a25096": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_33b3f2ed76434c13b68809069903b693", "placeholder": "\u200b", "style": "IPY_MODEL_ae9c1384a8ef40a9b2a32844f1646716", "tabbable": null, "tooltip": null, "value": "100%"}}, "e7fe152978d2459c9cf82d8f9f257479": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a6544264d0d741aa81b06d6cf1050e84": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "38bdea7d9b64452ea8420690243965cf": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e7fe152978d2459c9cf82d8f9f257479", "placeholder": "\u200b", "style": "IPY_MODEL_a6544264d0d741aa81b06d6cf1050e84", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200722.25it/s]"}}, "217459b8733641fda6a21621dc147583": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ef67630185584553867488aa7abe9fa9": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_1dfb46a04fe14273b07f0621a7a25096", "IPY_MODEL_69da672452a94ea68b438b3be68f1a24", "IPY_MODEL_38bdea7d9b64452ea8420690243965cf"], "layout": "IPY_MODEL_217459b8733641fda6a21621dc147583", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Main_Demo.ipynb b/generated/demos/Main_Demo.ipynb
index f8521a854..ac34c5192 100644
--- a/generated/demos/Main_Demo.ipynb
+++ b/generated/demos/Main_Demo.ipynb
@@ -48,10 +48,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:30.391298Z",
-     "iopub.status.busy": "2024-11-19T14:50:30.390806Z",
-     "iopub.status.idle": "2024-11-19T14:50:30.401467Z",
-     "shell.execute_reply": "2024-11-19T14:50:30.400992Z"
+     "iopub.execute_input": "2024-12-14T01:01:21.187448Z",
+     "iopub.status.busy": "2024-12-14T01:01:21.187251Z",
+     "iopub.status.idle": "2024-12-14T01:01:21.197652Z",
+     "shell.execute_reply": "2024-12-14T01:01:21.197226Z"
     }
    },
    "outputs": [],
@@ -89,10 +89,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:30.403503Z",
-     "iopub.status.busy": "2024-11-19T14:50:30.403156Z",
-     "iopub.status.idle": "2024-11-19T14:50:30.522258Z",
-     "shell.execute_reply": "2024-11-19T14:50:30.521707Z"
+     "iopub.execute_input": "2024-12-14T01:01:21.199828Z",
+     "iopub.status.busy": "2024-12-14T01:01:21.199477Z",
+     "iopub.status.idle": "2024-12-14T01:01:21.320090Z",
+     "shell.execute_reply": "2024-12-14T01:01:21.319483Z"
     }
    },
    "outputs": [
@@ -119,28 +119,28 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:30.524205Z",
-     "iopub.status.busy": "2024-11-19T14:50:30.524032Z",
-     "iopub.status.idle": "2024-11-19T14:50:31.907684Z",
-     "shell.execute_reply": "2024-11-19T14:50:31.907066Z"
+     "iopub.execute_input": "2024-12-14T01:01:21.322347Z",
+     "iopub.status.busy": "2024-12-14T01:01:21.321942Z",
+     "iopub.status.idle": "2024-12-14T01:01:22.768775Z",
+     "shell.execute_reply": "2024-12-14T01:01:22.768119Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-4c03d420-fc38\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-48c8c518-76ab\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, Hello } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-4c03d420-fc38\",\n",
+       "      \"circuits-vis-48c8c518-76ab\",\n",
        "      Hello,\n",
        "      {\"name\": \"Neel\"}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7fcde9144950>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f0f58854a90>"
       ]
      },
      "execution_count": 3,
@@ -159,10 +159,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:31.910173Z",
-     "iopub.status.busy": "2024-11-19T14:50:31.909697Z",
-     "iopub.status.idle": "2024-11-19T14:50:32.202280Z",
-     "shell.execute_reply": "2024-11-19T14:50:32.201720Z"
+     "iopub.execute_input": "2024-12-14T01:01:22.771137Z",
+     "iopub.status.busy": "2024-12-14T01:01:22.770703Z",
+     "iopub.status.idle": "2024-12-14T01:01:23.078839Z",
+     "shell.execute_reply": "2024-12-14T01:01:23.078261Z"
     }
    },
    "outputs": [],
@@ -184,10 +184,10 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:32.204872Z",
-     "iopub.status.busy": "2024-11-19T14:50:32.204390Z",
-     "iopub.status.idle": "2024-11-19T14:50:33.678512Z",
-     "shell.execute_reply": "2024-11-19T14:50:33.678024Z"
+     "iopub.execute_input": "2024-12-14T01:01:23.081471Z",
+     "iopub.status.busy": "2024-12-14T01:01:23.081005Z",
+     "iopub.status.idle": "2024-12-14T01:01:24.683057Z",
+     "shell.execute_reply": "2024-12-14T01:01:24.682500Z"
     }
    },
    "outputs": [],
@@ -212,17 +212,17 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:33.680923Z",
-     "iopub.status.busy": "2024-11-19T14:50:33.680644Z",
-     "iopub.status.idle": "2024-11-19T14:50:33.684402Z",
-     "shell.execute_reply": "2024-11-19T14:50:33.683888Z"
+     "iopub.execute_input": "2024-12-14T01:01:24.685764Z",
+     "iopub.status.busy": "2024-12-14T01:01:24.685314Z",
+     "iopub.status.idle": "2024-12-14T01:01:24.689311Z",
+     "shell.execute_reply": "2024-12-14T01:01:24.688764Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<torch.autograd.grad_mode.set_grad_enabled at 0x7fcdda667510>"
+       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f0f50f9df90>"
       ]
      },
      "execution_count": 6,
@@ -246,10 +246,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:33.686464Z",
-     "iopub.status.busy": "2024-11-19T14:50:33.686141Z",
-     "iopub.status.idle": "2024-11-19T14:50:33.690741Z",
-     "shell.execute_reply": "2024-11-19T14:50:33.690277Z"
+     "iopub.execute_input": "2024-12-14T01:01:24.691560Z",
+     "iopub.status.busy": "2024-12-14T01:01:24.691109Z",
+     "iopub.status.idle": "2024-12-14T01:01:24.695875Z",
+     "shell.execute_reply": "2024-12-14T01:01:24.695298Z"
     }
    },
    "outputs": [],
@@ -301,10 +301,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:33.692834Z",
-     "iopub.status.busy": "2024-11-19T14:50:33.692451Z",
-     "iopub.status.idle": "2024-11-19T14:50:33.695363Z",
-     "shell.execute_reply": "2024-11-19T14:50:33.694823Z"
+     "iopub.execute_input": "2024-12-14T01:01:24.697925Z",
+     "iopub.status.busy": "2024-12-14T01:01:24.697623Z",
+     "iopub.status.idle": "2024-12-14T01:01:24.700565Z",
+     "shell.execute_reply": "2024-12-14T01:01:24.700031Z"
     }
    },
    "outputs": [],
@@ -317,10 +317,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:33.697550Z",
-     "iopub.status.busy": "2024-11-19T14:50:33.697155Z",
-     "iopub.status.idle": "2024-11-19T14:50:34.923885Z",
-     "shell.execute_reply": "2024-11-19T14:50:34.923256Z"
+     "iopub.execute_input": "2024-12-14T01:01:24.702501Z",
+     "iopub.status.busy": "2024-12-14T01:01:24.702210Z",
+     "iopub.status.idle": "2024-12-14T01:01:25.742888Z",
+     "shell.execute_reply": "2024-12-14T01:01:25.742226Z"
     }
    },
    "outputs": [
@@ -354,10 +354,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:34.926227Z",
-     "iopub.status.busy": "2024-11-19T14:50:34.925851Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.241882Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.241341Z"
+     "iopub.execute_input": "2024-12-14T01:01:25.745568Z",
+     "iopub.status.busy": "2024-12-14T01:01:25.745062Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.063199Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.062608Z"
     }
    },
    "outputs": [
@@ -399,10 +399,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.244290Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.243945Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.379312Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.378687Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.065580Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.065191Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.204404Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.203778Z"
     }
    },
    "outputs": [
@@ -440,10 +440,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.381952Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.381607Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.386059Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.385480Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.206966Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.206475Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.210963Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.210505Z"
     }
    },
    "outputs": [
@@ -468,10 +468,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.388086Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.387770Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.479915Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.479362Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.212839Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.212666Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.324357Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.323744Z"
     }
    },
    "outputs": [
@@ -485,18 +485,18 @@
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-02b5f1a2-ff46\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-ac0a9437-0fc4\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-02b5f1a2-ff46\",\n",
+       "      \"circuits-vis-ac0a9437-0fc4\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"<|endoftext|>\", \"Natural\", \" language\", \" processing\", \" tasks\", \",\", \" such\", \" as\", \" question\", \" answering\", \",\", \" machine\", \" translation\", \",\", \" reading\", \" comprehension\", \",\", \" and\", \" summar\", \"ization\", \",\", \" are\", \" typically\", \" approached\", \" with\", \" supervised\", \" learning\", \" on\", \" tasks\", \"pe\", \"cific\", \" datasets\", \".\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7fcdd9d9c950>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f1034184810>"
       ]
      },
      "execution_count": 13,
@@ -509,6 +509,33 @@
     "cv.attention.attention_patterns(tokens=gpt2_str_tokens, attention=attention_pattern)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this case, we only wanted the layer 0 attention patterns, but we are storing the internal activations from all locations in the model. It's convenient to have access to all activations, but this can be prohibitively expensive for memory use with larger models, batch sizes, or sequence lengths. In addition, we don't need to do the full forward pass through the model to collect layer 0 attention patterns. The following cell will collect only the layer 0 attention patterns and stop the forward pass at layer 1, requiring far less memory and compute."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-12-14T01:01:26.326833Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.326375Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.344132Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.343488Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "attn_hook_name = \"blocks.0.attn.hook_pattern\"\n",
+    "attn_layer = 0\n",
+    "_, gpt2_attn_cache = model.run_with_cache(gpt2_tokens, remove_batch_dim=True, stop_at_layer=attn_layer + 1, names_filter=[attn_hook_name])\n",
+    "gpt2_attn = gpt2_attn_cache[attn_hook_name]\n",
+    "assert torch.equal(gpt2_attn, attention_pattern)"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -549,13 +576,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.481975Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.481677Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.733603Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.732943Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.346946Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.346593Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.595536Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.594923Z"
     }
    },
    "outputs": [
@@ -635,13 +662,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.735997Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.735668Z",
-     "iopub.status.idle": "2024-11-19T14:50:35.947083Z",
-     "shell.execute_reply": "2024-11-19T14:50:35.946474Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.598076Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.597740Z",
+     "iopub.status.idle": "2024-12-14T01:01:26.804705Z",
+     "shell.execute_reply": "2024-12-14T01:01:26.804124Z"
     }
    },
    "outputs": [
@@ -691,20 +718,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:35.949456Z",
-     "iopub.status.busy": "2024-11-19T14:50:35.949123Z",
-     "iopub.status.idle": "2024-11-19T14:50:55.652416Z",
-     "shell.execute_reply": "2024-11-19T14:50:55.651776Z"
+     "iopub.execute_input": "2024-12-14T01:01:26.807213Z",
+     "iopub.status.busy": "2024-12-14T01:01:26.806795Z",
+     "iopub.status.idle": "2024-12-14T01:01:46.682287Z",
+     "shell.execute_reply": "2024-12-14T01:01:46.681658Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8886699251a949f49e757b9fdc7ca890",
+       "model_id": "ddd91d21a45e49a8a90fd6693fc0fe90",
        "version_major": 2,
        "version_minor": 0
       },
@@ -758,13 +785,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:55.654866Z",
-     "iopub.status.busy": "2024-11-19T14:50:55.654529Z",
-     "iopub.status.idle": "2024-11-19T14:50:55.809395Z",
-     "shell.execute_reply": "2024-11-19T14:50:55.808754Z"
+     "iopub.execute_input": "2024-12-14T01:01:46.684826Z",
+     "iopub.status.busy": "2024-12-14T01:01:46.684383Z",
+     "iopub.status.idle": "2024-12-14T01:01:46.842465Z",
+     "shell.execute_reply": "2024-12-14T01:01:46.841862Z"
     }
    },
    "outputs": [
@@ -775,9 +802,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"9a4c3889-fbfe-43e2-a51a-d3c551c88f8d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"9a4c3889-fbfe-43e2-a51a-d3c551c88f8d\")) {                    Plotly.newPlot(                        \"9a4c3889-fbfe-43e2-a51a-d3c551c88f8d\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\")) {                    Plotly.newPlot(                        \"a86e9b27-7dd3-47fb-9141-10fa1ea8195f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('9a4c3889-fbfe-43e2-a51a-d3c551c88f8d');\n",
+       "var gd = document.getElementById('a86e9b27-7dd3-47fb-9141-10fa1ea8195f');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -851,13 +878,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:55.811657Z",
-     "iopub.status.busy": "2024-11-19T14:50:55.811338Z",
-     "iopub.status.idle": "2024-11-19T14:50:57.582424Z",
-     "shell.execute_reply": "2024-11-19T14:50:57.581809Z"
+     "iopub.execute_input": "2024-12-14T01:01:46.844857Z",
+     "iopub.status.busy": "2024-12-14T01:01:46.844522Z",
+     "iopub.status.idle": "2024-12-14T01:01:48.672019Z",
+     "shell.execute_reply": "2024-12-14T01:01:48.671432Z"
     }
    },
    "outputs": [
@@ -868,9 +895,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b221d135-41b1-42fe-b935-2c48e6f5a584\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b221d135-41b1-42fe-b935-2c48e6f5a584\")) {                    Plotly.newPlot(                        \"b221d135-41b1-42fe-b935-2c48e6f5a584\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.930997848510742,12.973093032836914,14.311477661132812,13.683721542358398,13.431002616882324,12.055716514587402,13.882497787475586,12.124085426330566,10.646708488464355,14.250656127929688,12.821611404418945,12.736834526062012,11.802390098571777,11.932378768920898,12.010724067687988,12.102657318115234,11.457335472106934,11.689569473266602,11.127102851867676,12.51093864440918,10.938901901245117,11.951817512512207,11.50601863861084,11.922638893127441,11.712303161621094,11.965950012207031,11.912420272827148,11.779227256774902,12.358148574829102,11.38554573059082,12.330202102661133,10.471684455871582,11.231712341308594,11.510209083557129,11.543947219848633,11.340058326721191,10.549806594848633,12.171330451965332,11.418136596679688,10.7109375,11.285449981689453,10.970927238464355,12.132821083068848,12.104890823364258,11.697572708129883,12.078733444213867,11.500871658325195,11.366061210632324,11.200624465942383,11.578920364379883,3.00230073928833,0.9546955227851868,0.7389153838157654,0.5628250241279602,0.4280739426612854,0.3580504059791565,0.5007835626602173,0.3580631613731384,0.24043424427509308,0.4543682038784027,0.5478570461273193,0.13366922736167908,0.13654688000679016,0.23078858852386475,0.1546991914510727,0.05507778376340866,0.35659217834472656,0.07221563160419464,0.050734974443912506,0.29230499267578125,0.04061192646622658,0.1435699164867401,0.3380035161972046,0.06990544497966766,0.18574923276901245,0.05935591459274292,0.08002041280269623,0.11634248495101929,0.09985627233982086,0.08987277746200562,0.04109250754117966,0.028509577736258507,0.052600979804992676,0.04117293283343315,0.08193311840295792,0.02876918390393257,0.011056417599320412,0.10547198355197906,0.13078591227531433,0.1317104995250702,0.033281438052654266,0.02778303623199463,0.03573056310415268,0.047907400876283646,0.0208450797945261,0.1270831674337387,0.07031372934579849,0.06472618877887726,0.0732506588101387],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\")) {                    Plotly.newPlot(                        \"d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.934269905090332,14.23181438446045,11.729150772094727,15.214762687683105,13.592933654785156,14.211908340454102,13.205050468444824,13.07355785369873,11.535038948059082,12.773419380187988,13.354753494262695,12.027746200561523,12.879249572753906,11.845925331115723,12.831506729125977,11.792170524597168,12.737360000610352,12.617140769958496,12.619315147399902,12.312422752380371,11.479436874389648,12.757272720336914,11.545267105102539,12.173948287963867,11.990314483642578,11.905129432678223,11.678251266479492,11.099139213562012,10.786980628967285,12.338418960571289,11.62226676940918,11.612360000610352,10.657169342041016,11.615835189819336,11.122636795043945,10.654256820678711,11.861252784729004,11.142145156860352,11.718962669372559,11.342132568359375,11.637442588806152,11.20296859741211,12.066685676574707,10.65315055847168,11.302213668823242,11.483949661254883,10.655682563781738,11.83389949798584,11.766695976257324,11.974489212036133,3.564096450805664,1.2277376651763916,0.9820038676261902,1.200751543045044,0.4091821312904358,0.5843209624290466,0.3591119647026062,0.43036898970603943,0.24223056435585022,0.44049954414367676,0.41071295738220215,0.21330496668815613,0.15890493988990784,0.31460171937942505,0.08366963267326355,0.2029363214969635,0.17958413064479828,0.1480543464422226,0.034813292324543,0.06144014745950699,0.09302718937397003,0.06429728120565414,0.4040629267692566,0.12488675117492676,0.08299500495195389,0.02960968390107155,0.01776202768087387,0.03346758708357811,0.10293184220790863,0.0828002542257309,0.008384309709072113,0.031925592571496964,0.1624717116355896,0.05249609425663948,0.02139073610305786,0.03643417730927467,0.07777489721775055,0.1070786863565445,0.03805286064743996,0.11206188052892685,0.1607268750667572,0.13429032266139984,0.16408373415470123,0.05713384598493576,0.12058018147945404,0.09626038372516632,0.04750080779194832,0.1064792275428772,0.15790846943855286],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('b221d135-41b1-42fe-b935-2c48e6f5a584');\n",
+       "var gd = document.getElementById('d81bb9d3-dd8a-47ab-a254-a6ee2bc9adcc');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -933,13 +960,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:57.584549Z",
-     "iopub.status.busy": "2024-11-19T14:50:57.584372Z",
-     "iopub.status.idle": "2024-11-19T14:50:58.759902Z",
-     "shell.execute_reply": "2024-11-19T14:50:58.759335Z"
+     "iopub.execute_input": "2024-12-14T01:01:48.674292Z",
+     "iopub.status.busy": "2024-12-14T01:01:48.673943Z",
+     "iopub.status.idle": "2024-12-14T01:01:49.886399Z",
+     "shell.execute_reply": "2024-12-14T01:01:49.885883Z"
     }
    },
    "outputs": [
@@ -950,9 +977,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"f61e6828-f047-4b78-be4c-94c6a5a16b3b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f61e6828-f047-4b78-be4c-94c6a5a16b3b\")) {                    Plotly.newPlot(                        \"f61e6828-f047-4b78-be4c-94c6a5a16b3b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010005488060414791,0.00010679646220523864,0.011149473488330841,1.6619707139398088e-06,0.00023922690888866782,0.00015111741959117353,0.009801512584090233,0.0007387498626485467,0.009316714480519295,0.009042520076036453,0.006868044380098581,0.01548066921532154],[0.0008192680543288589,0.00046259158989414573,0.0020540570840239525,0.014562157914042473,0.004917391575872898,0.01147772278636694,0.01584085077047348,0.01319159660488367,0.012704173102974892,0.015902359038591385,0.006454707123339176,0.0005202066386118531],[0.005110130179673433,0.018723005428910255,0.0012636644532904029,0.0013060116907581687,0.013963320292532444,0.0018745333654806018,0.004315933212637901,0.008237375877797604,0.0035762235056608915,0.0011971687199547887,0.0006931528332643211,0.010485384613275528],[0.015022697858512402,0.0073833102360367775,0.0018157829763367772,0.01069782767444849,0.021013276651501656,0.012958213686943054,0.001296360045671463,0.0009169265395030379,0.004911302588880062,0.011581930331885815,0.007676774635910988,0.00595051096752286],[0.015315228141844273,0.01408644113689661,0.014493301510810852,0.009663796983659267,0.018403636291623116,0.013842382468283176,0.00839169230312109,0.0018653222359716892,0.016032662242650986,0.013998506590723991,0.018994230777025223,7.747924080092616e-09],[0.4260586202144623,0.8949548602104187,0.01510448008775711,0.007229045499116182,0.012087161652743816,0.9142396450042725,0.01206196192651987,0.01813812553882599,0.02615417167544365,0.027872497215867043,0.018886346369981766,0.017591215670108795],[0.01042243279516697,0.016652364283800125,0.01844092272222042,0.014439660124480724,0.023017292842268944,0.011790945194661617,0.029514683410525322,0.010329627431929111,0.011278615333139896,0.904080867767334,0.03695948049426079,0.013053759932518005],[0.010848619975149632,0.18712441623210907,0.8447970151901245,0.018071545287966728,0.01874406263232231,0.01866166852414608,0.04627839848399162,0.09087574481964111,0.017286069691181183,0.019067486748099327,0.9059284329414368,0.05858321860432625],[0.017044229432940483,0.39027440547943115,0.015756050124764442,0.04679980129003525,0.018212294206023216,0.013268682174384594,0.1567089557647705,0.013300132006406784,0.0325496643781662,0.032548386603593826,0.06684791296720505,0.019402673467993736],[0.253298819065094,0.18304164707660675,0.10468680411577225,0.012513653375208378,0.0902019739151001,0.026693832129240036,0.4520256519317627,0.029920928180217743,0.05179687589406967,0.476281076669693,0.01758650131523609,0.04082362726330757],[0.345258504152298,0.5076920390129089,0.0380970723927021,0.14610175788402557,0.06365255266427994,0.01531550195068121,0.2981257736682892,0.48563432693481445,0.04946523532271385,0.016827359795570374,0.1584998071193695,0.2598787248134613],[0.017348697409033775,0.05146760120987892,0.03453077748417854,0.009557337500154972,0.0365176685154438,0.09732990711927414,0.0516226589679718,0.06467494368553162,0.008705069310963154,0.30858248472213745,0.38851720094680786,0.02194729819893837]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\")) {                    Plotly.newPlot(                        \"b2a3ed8a-ba81-4aac-a1a1-60e16774f655\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010606949217617512,0.00013902947830501944,0.01081207673996687,5.408500101111713e-07,0.00022116937907412648,7.771956006763503e-05,0.00958997942507267,0.0009253817261196673,0.009193798527121544,0.009053227491676807,0.006911635398864746,0.014761571772396564],[0.0008132570656016469,0.0004268861666787416,0.002124645747244358,0.01425299420952797,0.005162382964044809,0.010268471203744411,0.015934234485030174,0.013339280150830746,0.012906215153634548,0.016355415806174278,0.006486278492957354,0.0005485203000716865],[0.004910458344966173,0.019308894872665405,0.0012424595188349485,0.0010258526308462024,0.013039778918027878,0.0015818236861377954,0.00429101288318634,0.008304944261908531,0.003105038544163108,0.000971563218627125,0.0007218025857582688,0.010301434434950352],[0.015241926535964012,0.006064379122108221,0.00234038638882339,0.01002125907689333,0.021940935403108597,0.01423431932926178,0.0017911610193550587,0.0017028121510520577,0.005022364668548107,0.012421459890902042,0.00839681550860405,0.005942116491496563],[0.015156026929616928,0.013609741814434528,0.01541218627244234,0.00972918700426817,0.018237268552184105,0.013402285054326057,0.009189526550471783,0.0018425184534862638,0.015492293052375317,0.013836968690156937,0.0181787870824337,3.1250738885546525e-08],[0.42406418919563293,0.9055006504058838,0.015690578147768974,0.01017544511705637,0.01198639441281557,0.9278296828269958,0.012438608333468437,0.018412206321954727,0.027197834104299545,0.028249241411685944,0.019754743203520775,0.018243789672851562],[0.012461011298000813,0.01634187251329422,0.019040238112211227,0.015691598877310753,0.02333749644458294,0.012633594684302807,0.029116196557879448,0.012918342836201191,0.009929743595421314,0.92023104429245,0.035106346011161804,0.015322730876505375],[0.011738991364836693,0.17485056817531586,0.8415674567222595,0.01858351193368435,0.018508559092879295,0.018142390996217728,0.04514370858669281,0.08067239820957184,0.017961328849196434,0.019256191328167915,0.9187174439430237,0.052430037409067154],[0.01744176633656025,0.3839356303215027,0.01593034528195858,0.0451587438583374,0.018434638157486916,0.01456737332046032,0.14198516309261322,0.014199082739651203,0.03386015444993973,0.0334649458527565,0.06564274430274963,0.02554652839899063],[0.2423967868089676,0.19136165082454681,0.1043233796954155,0.012038059532642365,0.0845528319478035,0.027191946282982826,0.4313036799430847,0.028380122035741806,0.04645952582359314,0.4677841365337372,0.017513224855065346,0.03886174410581589],[0.3296402394771576,0.48611578345298767,0.0376255176961422,0.14191953837871552,0.06014999374747276,0.01589331030845642,0.28858682513237,0.479020357131958,0.05016819015145302,0.015279495157301426,0.14848238229751587,0.25642627477645874],[0.01643357053399086,0.05329973250627518,0.035110954195261,0.011042965576052666,0.03575966879725456,0.10564529150724411,0.048889338970184326,0.06571993976831436,0.009506475180387497,0.30028679966926575,0.3763805329799652,0.025962304323911667]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f61e6828-f047-4b78-be4c-94c6a5a16b3b');\n",
+       "var gd = document.getElementById('b2a3ed8a-ba81-4aac-a1a1-60e16774f655');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1024,31 +1051,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:58.761978Z",
-     "iopub.status.busy": "2024-11-19T14:50:58.761804Z",
-     "iopub.status.idle": "2024-11-19T14:50:58.969901Z",
-     "shell.execute_reply": "2024-11-19T14:50:58.969330Z"
+     "iopub.execute_input": "2024-12-14T01:01:49.888604Z",
+     "iopub.status.busy": "2024-12-14T01:01:49.888277Z",
+     "iopub.status.idle": "2024-12-14T01:01:50.092655Z",
+     "shell.execute_reply": "2024-12-14T01:01:50.091997Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-ec824f3e-cfa6\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-61a73e43-0cbc\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-ec824f3e-cfa6\",\n",
+       "      \"circuits-vis-61a73e43-0cbc\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\", \"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7fcdcedb51d0>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f0f449f7390>"
       ]
      },
      "metadata": {},
@@ -1114,20 +1141,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:50:58.972572Z",
-     "iopub.status.busy": "2024-11-19T14:50:58.972188Z",
-     "iopub.status.idle": "2024-11-19T14:51:02.858894Z",
-     "shell.execute_reply": "2024-11-19T14:51:02.858293Z"
+     "iopub.execute_input": "2024-12-14T01:01:50.095371Z",
+     "iopub.status.busy": "2024-12-14T01:01:50.094972Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.259149Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.258602Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "504b41851a444c31b688e5e8b063add5",
+       "model_id": "2b653bed330e47bfaf1eb97604ffb148",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1141,7 +1168,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8efd0c3ea3bc4e5d8c1b966bff063c3a",
+       "model_id": "2f95bda34e124bdb8ad42b97968c2560",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1155,7 +1182,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5df370f098ea41fba5fb462b17adbe25",
+       "model_id": "1bfe4dc8f9d64ed3a70a0e6e41b61d34",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1169,7 +1196,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d5f0c8842a284fdda4dfa3206605c35f",
+       "model_id": "363a53f62bad40f38d9b57415fcdd27a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1183,7 +1210,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a678c766e93d424daa0150a52a8f38a2",
+       "model_id": "baa802cac1bf4a77bd2d6f2da4e0c206",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1197,7 +1224,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "da7af55f6bb5479b852a41df346f73ee",
+       "model_id": "85239cb8d6ba4d288b001290e919d816",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1223,13 +1250,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:02.861278Z",
-     "iopub.status.busy": "2024-11-19T14:51:02.860895Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.465993Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.465415Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.261641Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.261263Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.887545Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.886951Z"
     }
    },
    "outputs": [
@@ -1240,9 +1267,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2b76182f-a588-4611-91f1-3158be36fbea\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2b76182f-a588-4611-91f1-3158be36fbea\")) {                    Plotly.newPlot(                        \"2b76182f-a588-4611-91f1-3158be36fbea\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.009826192632317543,0.00015374009672086686,0.012172531336545944,6.603482688660733e-06,0.000805103627499193,2.0451569071155973e-05,0.008835355751216412,0.0015585398068651557,0.008454771712422371,0.009232211858034134,0.009152095764875412,0.016232741996645927],[0.0028829516377300024,0.01789030432701111,0.0022680708207190037,0.0004275149549357593,0.012768547050654888,0.0016649786848574877,0.0051553091034293175,0.015345298685133457,0.002696307608857751,8.173943206202239e-05,0.003935561515390873,0.014916213229298592],[0.007932284846901894,0.006039753090590239,0.012212091125547886,0.0025668356101959944,0.019200311973690987,0.004120599944144487,0.009461229667067528,0.0011959228431805968,0.017644984647631645,0.013755349442362785,0.021033549681305885,6.036892652439008e-10],[0.006974583957344294,0.22314414381980896,0.8409926295280457,0.01541733369231224,0.017725780606269836,0.014554478228092194,0.019251752644777298,0.18048787117004395,0.016629252582788467,0.015987560153007507,0.9273980855941772,0.46241772174835205],[0.2584259510040283,0.21280966699123383,0.0862935483455658,0.011735445819795132,0.07724905014038086,0.02455822005867958,0.6039906144142151,0.02535823918879032,0.06264935433864594,0.6323530077934265,0.017074570059776306,0.055065713822841644],[0.021564770489931107,0.07452036440372467,0.052915140986442566,0.011579645797610283,0.034101374447345734,0.17451111972332,0.07449180632829666,0.08992660045623779,0.008418177254498005,0.43939897418022156,0.17041026055812836,0.02964613400399685]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e151a52d-4559-414d-bbd0-4a08a83b57c6\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e151a52d-4559-414d-bbd0-4a08a83b57c6\")) {                    Plotly.newPlot(                        \"e151a52d-4559-414d-bbd0-4a08a83b57c6\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010150859132409096,0.00021067557099740952,0.011675543151795864,5.645640158036258e-06,0.0009093668777495623,1.0044651389762294e-05,0.008776976726949215,0.0014797636540606618,0.008503900840878487,0.009458577260375023,0.009176097810268402,0.015341502614319324],[0.0030316568445414305,0.018613651394844055,0.003521384671330452,0.0002105814783135429,0.012771016918122768,0.002262213034555316,0.005199276376515627,0.015398663468658924,0.001596646150574088,7.471267599612474e-05,0.004113694187253714,0.014634193852543831],[0.00835011899471283,0.005591983906924725,0.01345061045140028,0.0016680879052728415,0.019946973770856857,0.004370617680251598,0.009695201180875301,0.001158412080258131,0.01791170984506607,0.01292894221842289,0.021461093798279762,5.112248818717191e-11],[0.005671565420925617,0.20814286172389984,0.8488407135009766,0.01734151318669319,0.017860308289527893,0.014111736789345741,0.020909568294882774,0.1804729551076889,0.01673540472984314,0.0179011020809412,0.9288043975830078,0.4491482973098755],[0.26883465051651,0.23360644280910492,0.08689460158348083,0.011528643779456615,0.07701855152845383,0.024650055915117264,0.6229627132415771,0.025258244946599007,0.06073903664946556,0.6466246247291565,0.017515525221824646,0.054975464940071106],[0.021400248631834984,0.07713880389928818,0.05230269953608513,0.013922988437116146,0.032270368188619614,0.18037253618240356,0.07306526601314545,0.0951840952038765,0.007617602590471506,0.4484187066555023,0.17469297349452972,0.03143536299467087]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('2b76182f-a588-4611-91f1-3158be36fbea');\n",
+       "var gd = document.getElementById('e151a52d-4559-414d-bbd0-4a08a83b57c6');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1421,13 +1448,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.468239Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.468061Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.471887Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.471323Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.890023Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.889670Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.893651Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.893084Z"
     }
    },
    "outputs": [
@@ -1465,13 +1492,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.473939Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.473612Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.477098Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.476508Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.895827Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.895486Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.899011Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.898465Z"
     }
    },
    "outputs": [
@@ -1510,13 +1537,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.479110Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.478747Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.550143Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.549589Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.901258Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.900920Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.970759Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.970165Z"
     }
    },
    "outputs": [
@@ -1607,13 +1634,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.552590Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.552211Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.559932Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.559457Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.973146Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.972816Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.980352Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.979763Z"
     }
    },
    "outputs": [],
@@ -1624,13 +1651,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.562086Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.561742Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.568245Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.567697Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.982600Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.982257Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.987655Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.987079Z"
     }
    },
    "outputs": [
@@ -1705,13 +1732,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.570337Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.570032Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.574297Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.573852Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.989809Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.989465Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.993884Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.993446Z"
     }
    },
    "outputs": [
@@ -1773,13 +1800,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.576188Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.576022Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.580116Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.579558Z"
+     "iopub.execute_input": "2024-12-14T01:01:53.996070Z",
+     "iopub.status.busy": "2024-12-14T01:01:53.995639Z",
+     "iopub.status.idle": "2024-12-14T01:01:53.999873Z",
+     "shell.execute_reply": "2024-12-14T01:01:53.999303Z"
     }
    },
    "outputs": [
@@ -1806,13 +1833,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.582085Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.581919Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.585421Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.584982Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.002099Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.001703Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.005196Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.004759Z"
     }
    },
    "outputs": [
@@ -1846,13 +1873,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.587305Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.587141Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.590864Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.590431Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.007056Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.006893Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.010707Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.010246Z"
     }
    },
    "outputs": [
@@ -1889,13 +1916,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.592732Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.592566Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.683001Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.682385Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.013081Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.012542Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.100258Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.099682Z"
     }
    },
    "outputs": [
@@ -1903,7 +1930,13 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n",
+      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
       "| The| probability: 11.98%\n"
      ]
     }
@@ -1929,13 +1962,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.685511Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.685154Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.688656Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.688081Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.102428Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.102246Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.105837Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.105312Z"
     }
    },
    "outputs": [
@@ -1967,13 +2000,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.690760Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.690446Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.694710Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.694253Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.107939Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.107614Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.111996Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.111534Z"
     }
    },
    "outputs": [
@@ -2000,13 +2033,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.696837Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.696515Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.700693Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.700256Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.114136Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.113818Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.118199Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.117648Z"
     }
    },
    "outputs": [
@@ -2039,13 +2072,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.702708Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.702384Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.706262Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.705711Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.120410Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.120080Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.124019Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.123552Z"
     }
    },
    "outputs": [
@@ -2090,13 +2123,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.708174Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.708011Z",
-     "iopub.status.idle": "2024-11-19T14:51:03.907520Z",
-     "shell.execute_reply": "2024-11-19T14:51:03.906876Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.126145Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.125812Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.343747Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.343133Z"
     }
    },
    "outputs": [
@@ -2118,14 +2151,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Logits shape without BOS - only 2 positions!"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      " torch.Size([1, 2, 50257])\n"
+      "Logits shape without BOS - only 2 positions! torch.Size([1, 2, 50257])\n"
      ]
     }
    ],
@@ -2150,13 +2176,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:03.909963Z",
-     "iopub.status.busy": "2024-11-19T14:51:03.909565Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.114947Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.114346Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.346431Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.346064Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.545626Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.545021Z"
     }
    },
    "outputs": [
@@ -2171,7 +2197,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Logit difference without BOS: 2.782\n"
+      "Logit difference without BOS: 2.782"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
      ]
     }
    ],
@@ -2196,13 +2229,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.117450Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.117053Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.121320Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.120826Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.548090Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.547737Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.551966Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.551372Z"
     }
    },
    "outputs": [
@@ -2256,13 +2289,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 41,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.123455Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.123110Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.128917Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.128472Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.554257Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.553915Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.559609Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.559136Z"
     }
    },
    "outputs": [
@@ -2301,13 +2334,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 42,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.131051Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.130734Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.135599Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.135148Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.561788Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.561461Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.566401Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.565937Z"
     }
    },
    "outputs": [
@@ -2346,13 +2379,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 42,
+   "execution_count": 43,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.137615Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.137304Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.142799Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.142224Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.568434Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.568104Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.573557Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.573073Z"
     }
    },
    "outputs": [
@@ -2388,13 +2421,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 44,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.144834Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.144668Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.148231Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.147746Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.575636Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.575308Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.578709Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.578240Z"
     }
    },
    "outputs": [
@@ -2438,13 +2471,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 45,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.150324Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.150004Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.157480Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.156919Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.580629Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.580461Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.587025Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.586507Z"
     }
    },
    "outputs": [
@@ -2463,13 +2496,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
+   "execution_count": 46,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.159729Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.159357Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.296907Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.296336Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.589336Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.588929Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.698460Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.697829Z"
     }
    },
    "outputs": [
@@ -2490,13 +2523,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 47,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.299250Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.298866Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.328956Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.328495Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.700835Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.700449Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.731325Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.730826Z"
     }
    },
    "outputs": [
@@ -2507,9 +2540,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"29f8dfbd-47fb-4149-ba30-739032475d3b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"29f8dfbd-47fb-4149-ba30-739032475d3b\")) {                    Plotly.newPlot(                        \"29f8dfbd-47fb-4149-ba30-739032475d3b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"458e8707-ccf3-4567-a8b0-f931c572b246\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"458e8707-ccf3-4567-a8b0-f931c572b246\")) {                    Plotly.newPlot(                        \"458e8707-ccf3-4567-a8b0-f931c572b246\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('29f8dfbd-47fb-4149-ba30-739032475d3b');\n",
+       "var gd = document.getElementById('458e8707-ccf3-4567-a8b0-f931c572b246');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2554,13 +2587,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 48,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.331107Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.330779Z",
-     "iopub.status.idle": "2024-11-19T14:51:04.360174Z",
-     "shell.execute_reply": "2024-11-19T14:51:04.359707Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.733451Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.733127Z",
+     "iopub.status.idle": "2024-12-14T01:01:54.763618Z",
+     "shell.execute_reply": "2024-12-14T01:01:54.762996Z"
     }
    },
    "outputs": [
@@ -2571,9 +2604,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"de463d3c-abc6-4c73-976f-c8458ad4e0e7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"de463d3c-abc6-4c73-976f-c8458ad4e0e7\")) {                    Plotly.newPlot(                        \"de463d3c-abc6-4c73-976f-c8458ad4e0e7\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"79eef653-2ecc-4731-aedd-e4b748085838\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"79eef653-2ecc-4731-aedd-e4b748085838\")) {                    Plotly.newPlot(                        \"79eef653-2ecc-4731-aedd-e4b748085838\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('de463d3c-abc6-4c73-976f-c8458ad4e0e7');\n",
+       "var gd = document.getElementById('79eef653-2ecc-4731-aedd-e4b748085838');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2617,13 +2650,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": 49,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:04.362207Z",
-     "iopub.status.busy": "2024-11-19T14:51:04.361867Z",
-     "iopub.status.idle": "2024-11-19T14:51:14.593048Z",
-     "shell.execute_reply": "2024-11-19T14:51:14.592428Z"
+     "iopub.execute_input": "2024-12-14T01:01:54.765962Z",
+     "iopub.status.busy": "2024-12-14T01:01:54.765604Z",
+     "iopub.status.idle": "2024-12-14T01:02:05.135799Z",
+     "shell.execute_reply": "2024-12-14T01:02:05.135140Z"
     }
    },
    "outputs": [
@@ -2642,13 +2675,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 50,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:14.595503Z",
-     "iopub.status.busy": "2024-11-19T14:51:14.595090Z",
-     "iopub.status.idle": "2024-11-19T14:51:15.129213Z",
-     "shell.execute_reply": "2024-11-19T14:51:15.128433Z"
+     "iopub.execute_input": "2024-12-14T01:02:05.138206Z",
+     "iopub.status.busy": "2024-12-14T01:02:05.137845Z",
+     "iopub.status.idle": "2024-12-14T01:02:05.685306Z",
+     "shell.execute_reply": "2024-12-14T01:02:05.684702Z"
     }
    },
    "outputs": [
@@ -2669,13 +2702,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 51,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:15.131668Z",
-     "iopub.status.busy": "2024-11-19T14:51:15.131283Z",
-     "iopub.status.idle": "2024-11-19T14:51:15.163437Z",
-     "shell.execute_reply": "2024-11-19T14:51:15.162925Z"
+     "iopub.execute_input": "2024-12-14T01:02:05.687795Z",
+     "iopub.status.busy": "2024-12-14T01:02:05.687450Z",
+     "iopub.status.idle": "2024-12-14T01:02:05.718903Z",
+     "shell.execute_reply": "2024-12-14T01:02:05.718334Z"
     }
    },
    "outputs": [
@@ -2686,9 +2719,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e218ffb2-72ea-46fa-b8e3-fe441e03c97b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e218ffb2-72ea-46fa-b8e3-fe441e03c97b\")) {                    Plotly.newPlot(                        \"e218ffb2-72ea-46fa-b8e3-fe441e03c97b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b62ac579-268a-4cb8-a4db-de9c144e5855\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b62ac579-268a-4cb8-a4db-de9c144e5855\")) {                    Plotly.newPlot(                        \"b62ac579-268a-4cb8-a4db-de9c144e5855\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e218ffb2-72ea-46fa-b8e3-fe441e03c97b');\n",
+       "var gd = document.getElementById('b62ac579-268a-4cb8-a4db-de9c144e5855');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2733,13 +2766,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 52,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:15.165808Z",
-     "iopub.status.busy": "2024-11-19T14:51:15.165426Z",
-     "iopub.status.idle": "2024-11-19T14:51:15.201349Z",
-     "shell.execute_reply": "2024-11-19T14:51:15.200782Z"
+     "iopub.execute_input": "2024-12-14T01:02:05.721386Z",
+     "iopub.status.busy": "2024-12-14T01:02:05.721038Z",
+     "iopub.status.idle": "2024-12-14T01:02:05.752513Z",
+     "shell.execute_reply": "2024-12-14T01:02:05.751952Z"
     }
    },
    "outputs": [
@@ -2750,9 +2783,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5\")) {                    Plotly.newPlot(                        \"dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"3c8c77e9-6054-4886-85c5-8868e8e7c01c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3c8c77e9-6054-4886-85c5-8868e8e7c01c\")) {                    Plotly.newPlot(                        \"3c8c77e9-6054-4886-85c5-8868e8e7c01c\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('dd6b1eb6-d18a-4e3a-abb8-9a4d06b38ff5');\n",
+       "var gd = document.getElementById('3c8c77e9-6054-4886-85c5-8868e8e7c01c');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2789,13 +2822,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 53,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:15.203499Z",
-     "iopub.status.busy": "2024-11-19T14:51:15.203331Z",
-     "iopub.status.idle": "2024-11-19T14:51:15.206826Z",
-     "shell.execute_reply": "2024-11-19T14:51:15.206300Z"
+     "iopub.execute_input": "2024-12-14T01:02:05.754764Z",
+     "iopub.status.busy": "2024-12-14T01:02:05.754419Z",
+     "iopub.status.idle": "2024-12-14T01:02:05.757816Z",
+     "shell.execute_reply": "2024-12-14T01:02:05.757251Z"
     }
    },
    "outputs": [
@@ -2833,20 +2866,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 54,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:15.208998Z",
-     "iopub.status.busy": "2024-11-19T14:51:15.208666Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.448233Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.447673Z"
+     "iopub.execute_input": "2024-12-14T01:02:05.759822Z",
+     "iopub.status.busy": "2024-12-14T01:02:05.759655Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.062119Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.061525Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ca0421e9c3074fe38697b4e54fe160b3",
+       "model_id": "ef67630185584553867488aa7abe9fa9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -2863,7 +2896,7 @@
        "'(CNN) President Barack Obama caught in embarrassing new scandal\\n\\nThe president, in an interview with the Financial Times, said that he did not know when he was caught on video talking about his wife, Chelsea, but he did know that she was a \"young woman\" and that he had been using her'"
       ]
      },
-     "execution_count": 53,
+     "execution_count": 54,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2919,13 +2952,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 55,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.450630Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.450294Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.455529Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.455052Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.064531Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.064357Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.069953Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.069480Z"
     }
    },
    "outputs": [],
@@ -2984,13 +3017,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 55,
+   "execution_count": 56,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.457650Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.457321Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.461481Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.460896Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.071912Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.071742Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.076151Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.075560Z"
     }
    },
    "outputs": [
@@ -3026,13 +3059,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 57,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.463519Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.463210Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.467125Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.466578Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.078214Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.078044Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.082026Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.081479Z"
     }
    },
    "outputs": [
@@ -3100,13 +3133,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 57,
+   "execution_count": 58,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.469248Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.468921Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.981903Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.981296Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.084310Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.083977Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.374838Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.374272Z"
     }
    },
    "outputs": [
@@ -3117,9 +3150,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"f75ed1fa-2d37-438f-a502-ed44a36bd3db\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f75ed1fa-2d37-438f-a502-ed44a36bd3db\")) {                    Plotly.newPlot(                        \"f75ed1fa-2d37-438f-a502-ed44a36bd3db\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"76053182-4b0b-44d5-9e6f-dc68cfcabb60\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"76053182-4b0b-44d5-9e6f-dc68cfcabb60\")) {                    Plotly.newPlot(                        \"76053182-4b0b-44d5-9e6f-dc68cfcabb60\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f75ed1fa-2d37-438f-a502-ed44a36bd3db');\n",
+       "var gd = document.getElementById('76053182-4b0b-44d5-9e6f-dc68cfcabb60');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3156,9 +3189,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"3bb32d65-c865-49b8-9cc7-92f6460f7bf6\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3bb32d65-c865-49b8-9cc7-92f6460f7bf6\")) {                    Plotly.newPlot(                        \"3bb32d65-c865-49b8-9cc7-92f6460f7bf6\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"22829199-cf37-49cf-8817-a42bc6f79448\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"22829199-cf37-49cf-8817-a42bc6f79448\")) {                    Plotly.newPlot(                        \"22829199-cf37-49cf-8817-a42bc6f79448\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('3bb32d65-c865-49b8-9cc7-92f6460f7bf6');\n",
+       "var gd = document.getElementById('22829199-cf37-49cf-8817-a42bc6f79448');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3195,9 +3228,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"37192a3b-da9b-4923-8d68-311c63c28642\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"37192a3b-da9b-4923-8d68-311c63c28642\")) {                    Plotly.newPlot(                        \"37192a3b-da9b-4923-8d68-311c63c28642\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"86fd24b0-a205-4e6c-8105-f79fcf57c539\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"86fd24b0-a205-4e6c-8105-f79fcf57c539\")) {                    Plotly.newPlot(                        \"86fd24b0-a205-4e6c-8105-f79fcf57c539\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('37192a3b-da9b-4923-8d68-311c63c28642');\n",
+       "var gd = document.getElementById('86fd24b0-a205-4e6c-8105-f79fcf57c539');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3234,9 +3267,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"d996cbd1-4f6e-452c-8efb-5d303fe6d5ab\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d996cbd1-4f6e-452c-8efb-5d303fe6d5ab\")) {                    Plotly.newPlot(                        \"d996cbd1-4f6e-452c-8efb-5d303fe6d5ab\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"4a6e81dc-ddca-460f-873f-0a213188ab89\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4a6e81dc-ddca-460f-873f-0a213188ab89\")) {                    Plotly.newPlot(                        \"4a6e81dc-ddca-460f-873f-0a213188ab89\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d996cbd1-4f6e-452c-8efb-5d303fe6d5ab');\n",
+       "var gd = document.getElementById('4a6e81dc-ddca-460f-873f-0a213188ab89');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3273,9 +3306,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"28df3ce6-a5fb-4e23-a85c-7ef4a03ed832\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"28df3ce6-a5fb-4e23-a85c-7ef4a03ed832\")) {                    Plotly.newPlot(                        \"28df3ce6-a5fb-4e23-a85c-7ef4a03ed832\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e2877398-410c-47b5-9f61-1fd97414d652\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e2877398-410c-47b5-9f61-1fd97414d652\")) {                    Plotly.newPlot(                        \"e2877398-410c-47b5-9f61-1fd97414d652\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('28df3ce6-a5fb-4e23-a85c-7ef4a03ed832');\n",
+       "var gd = document.getElementById('e2877398-410c-47b5-9f61-1fd97414d652');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3347,13 +3380,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
+   "execution_count": 59,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.984138Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.983812Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.987033Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.986468Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.377335Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.376955Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.380056Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.379541Z"
     }
    },
    "outputs": [],
@@ -3377,13 +3410,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 59,
+   "execution_count": 60,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.989023Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.988833Z",
-     "iopub.status.idle": "2024-11-19T14:51:17.992156Z",
-     "shell.execute_reply": "2024-11-19T14:51:17.991607Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.382096Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.381792Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.385346Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.384774Z"
     }
    },
    "outputs": [],
@@ -3412,13 +3445,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 60,
+   "execution_count": 61,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-11-19T14:51:17.994167Z",
-     "iopub.status.busy": "2024-11-19T14:51:17.993837Z",
-     "iopub.status.idle": "2024-11-19T14:51:18.023465Z",
-     "shell.execute_reply": "2024-11-19T14:51:18.022959Z"
+     "iopub.execute_input": "2024-12-14T01:02:08.387531Z",
+     "iopub.status.busy": "2024-12-14T01:02:08.387061Z",
+     "iopub.status.idle": "2024-12-14T01:02:08.419341Z",
+     "shell.execute_reply": "2024-12-14T01:02:08.418749Z"
     }
    },
    "outputs": [
@@ -3429,9 +3462,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"0ebddc76-ece2-4920-a2c7-60871b11afed\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"0ebddc76-ece2-4920-a2c7-60871b11afed\")) {                    Plotly.newPlot(                        \"0ebddc76-ece2-4920-a2c7-60871b11afed\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e8c48c58-6618-4eb6-8790-24b50ffe4409\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e8c48c58-6618-4eb6-8790-24b50ffe4409\")) {                    Plotly.newPlot(                        \"e8c48c58-6618-4eb6-8790-24b50ffe4409\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('0ebddc76-ece2-4920-a2c7-60871b11afed');\n",
+       "var gd = document.getElementById('e8c48c58-6618-4eb6-8790-24b50ffe4409');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3483,7 +3516,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.10"
+   "version": "3.11.11"
   },
   "vscode": {
    "interpreter": {
@@ -3493,51 +3526,83 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "051ba86a5dc04ae7a169b5a2f5d883db": {
+     "0093c35dd59c47adaed249b404d1a0b1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_3adaa7786f684ff6abb501929b91bfeb",
+       "placeholder": "​",
+       "style": "IPY_MODEL_236a3fe34dbb46b9a945a6c32dc4ed37",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 124/124 [00:00&lt;00:00, 21.8kB/s]"
       }
      },
-     "0813474f79c445e4969ff3384cba2b31": {
-      "model_module": "@jupyter-widgets/controls",
+     "0347d987e2964dd4a31348434040c0c7": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_d99e129cdb3147aa913c3c969c4c5910",
-       "max": 50.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_81593b1d4ed44438ac88b2d7a8b8cb96",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 50.0
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "099201fc3592463a99cefa3adf51b80e": {
+     "0f1f32e4ccb44e77b4a1aab830f601a5": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -3552,15 +3617,31 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_a452f9720b0141f18f4f1585cfbcf102",
+       "layout": "IPY_MODEL_9fce92e16f574800b2a9add4b6c10ef8",
        "placeholder": "​",
-       "style": "IPY_MODEL_051ba86a5dc04ae7a169b5a2f5d883db",
+       "style": "IPY_MODEL_bfdbeaa99bc942abb6a71e94629483ab",
        "tabbable": null,
        "tooltip": null,
-       "value": "100%"
+       "value": "tokenizer.json: 100%"
+      }
+     },
+     "16df306b166340878aae8ee0d7d2af94": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "0a116157d07c4ab1801aae54d0153bfd": {
+     "19da29c1b3cf4fae9710cd70f0baf271": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3613,7 +3694,31 @@
        "width": null
       }
      },
-     "12c5e2e95f084803834b82ff36d5a9aa": {
+     "1bfe4dc8f9d64ed3a70a0e6e41b61d34": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_d671580b566f41a3b09b9e31d0a97129",
+        "IPY_MODEL_f50d3b1ce11a44d8b5801bdd32a86fa8",
+        "IPY_MODEL_a3a46ce06ec04834810c4ab96c959684"
+       ],
+       "layout": "IPY_MODEL_7129d439a9d549d48802b08586b8c24f",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "1c150b40a42e403b848a74fd683a4788": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3666,152 +3771,30 @@
        "width": null
       }
      },
-     "14879e61652b40c5ba52609fd5646866": {
+     "1dfb46a04fe14273b07f0621a7a25096": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "1510c65a28504ad5b352dcd079536344": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_fcc3cac4665349559e159dd7b1be5c7f",
-       "placeholder": "​",
-       "style": "IPY_MODEL_35de1507ddfa4b87b3e78c74855788cd",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 4.50kB/s]"
-      }
-     },
-     "17bb4044cad04889b7a3185f7cca5d61": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_39fcd78619f74e7fa34ba3c717ecde79",
-       "placeholder": "​",
-       "style": "IPY_MODEL_8d0044c9d9594cdc99a744683335c85f",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "merges.txt: 100%"
-      }
-     },
-     "181e4adfc3bc45cd965a56602246337a": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_8aa3d272612246cb9ffad05643ac4d7d",
+       "layout": "IPY_MODEL_33b3f2ed76434c13b68809069903b693",
        "placeholder": "​",
-       "style": "IPY_MODEL_537c952c420e4ae1aea0035c609ce165",
+       "style": "IPY_MODEL_ae9c1384a8ef40a9b2a32844f1646716",
        "tabbable": null,
        "tooltip": null,
        "value": "100%"
       }
      },
-     "1c96e44af3b24a34aaff4045e496a27d": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_d027f2bb85564a00bf60cfe08b9b71ff",
-       "max": 12.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_14879e61652b40c5ba52609fd5646866",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 12.0
-      }
-     },
-     "1e61511b645b4c07aabc82cde43f0707": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "1ed9a2e74124405fb3abc69c91bd1ea6": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "1f2d34420a9d422db4c26669c9eece1b": {
+     "217459b8733641fda6a21621dc147583": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3864,7 +3847,7 @@
        "width": null
       }
      },
-     "1fa06c806c4a4980bbe87f0746c9a335": {
+     "22fa2830a0ab4ae4bb190db8ab08f59c": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3917,30 +3900,78 @@
        "width": null
       }
      },
-     "27310f32e9974dcd94aa2b9c49c1e6e7": {
+     "236a3fe34dbb46b9a945a6c32dc4ed37": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_0a116157d07c4ab1801aae54d0153bfd",
-       "placeholder": "​",
-       "style": "IPY_MODEL_e75a1f080c744b1c8939f0abb8e708fc",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 12/12 [00:19&lt;00:00,  1.64s/it]"
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "25f38d8b724d481397c21ffd4e2ae408": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/base",
+       "_model_module_version": "2.0.0",
+       "_model_name": "LayoutModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "2ce16701f86e465fa67e824fed52d560": {
+     "2974465a897b4ac9ba7d573f8e1ea5f8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3993,46 +4024,55 @@
        "width": null
       }
      },
-     "2eb8521bbd3d441180ebfef6c4e91184": {
+     "2b653bed330e47bfaf1eb97604ffb148": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_9bd1760890dd4e298154efe56e5a757d",
-       "placeholder": "​",
-       "style": "IPY_MODEL_4eb29c030d0b4615a334a1bf3f16ded1",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_c70847ad8e5543748ffbab29446b51c1",
+        "IPY_MODEL_4116100d429b4b9faf99f919b8cba2fd",
+        "IPY_MODEL_6304f591778f49058f9403f77c821c56"
+       ],
+       "layout": "IPY_MODEL_1c150b40a42e403b848a74fd683a4788",
        "tabbable": null,
-       "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 2.48MB/s]"
+       "tooltip": null
       }
      },
-     "3184aa14ca4b42eca4a28d1cdd508bea": {
+     "2f95bda34e124bdb8ad42b97968c2560": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_3a73a2ce3d684e57af5ead4d970bb546",
+        "IPY_MODEL_45071b25ff23496094a139022c8edc54",
+        "IPY_MODEL_0093c35dd59c47adaed249b404d1a0b1"
+       ],
+       "layout": "IPY_MODEL_2fbd1d8558174aaeae9046da37a2dc08",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "34b16debba6742959118694ecadd4e80": {
+     "2fbd1d8558174aaeae9046da37a2dc08": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4085,7 +4125,7 @@
        "width": null
       }
      },
-     "35de1507ddfa4b87b3e78c74855788cd": {
+     "30e033f4a4a745f99416fa6c47e6b1b8": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4103,33 +4143,7 @@
        "text_color": null
       }
      },
-     "397d196b08ea47da94f5d96dd6276340": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_d23203daf6b546699057c5635a148bad",
-       "max": 124.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_3184aa14ca4b42eca4a28d1cdd508bea",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 124.0
-      }
-     },
-     "39fcd78619f74e7fa34ba3c717ecde79": {
+     "3383a9989d6a4375bccecc436f7df6c0": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4182,7 +4196,7 @@
        "width": null
       }
      },
-     "40f00596573845cfb4f6ee3bf57fcf62": {
+     "33b3f2ed76434c13b68809069903b693": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4235,51 +4249,60 @@
        "width": null
       }
      },
-     "483d3536ea8644e0932ff376a08a7509": {
-      "model_module": "@jupyter-widgets/controls",
+     "355529cac414494c83e338e54c92a4cc": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_b5cfa8bab51f4768968002c31b9c4ccc",
-       "max": 1042301.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_5168126e6e2640259859e571cf02ac2b",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1042301.0
-      }
-     },
-     "4eb29c030d0b4615a334a1bf3f16ded1": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "504b41851a444c31b688e5e8b063add5": {
+     "363a53f62bad40f38d9b57415fcdd27a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -4294,68 +4317,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_df6f557d81e94e78b53b3b8791c846c4",
-        "IPY_MODEL_ed792737675641488478dfe9e706af53",
-        "IPY_MODEL_e8077a539c2f4fe7a0bc96e4cd70fa7e"
+        "IPY_MODEL_98db25d1775c4574b9702ae9d27f9d76",
+        "IPY_MODEL_975c4858e139485f8f0aa84771e564a4",
+        "IPY_MODEL_3b38cd8e7a4749a49d9442a541dd6d69"
        ],
-       "layout": "IPY_MODEL_9d118b82362542f59e0227f8f93cf219",
+       "layout": "IPY_MODEL_a6d329232ced49958c43f07f02862115",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "5054bc9f6dc94e5a938d3b2a410a235f": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "5168126e6e2640259859e571cf02ac2b": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "537c952c420e4ae1aea0035c609ce165": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "5adce76b5f7845b087083c2207ab56d3": {
+     "38bdea7d9b64452ea8420690243965cf": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -4370,39 +4341,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_718432154ef24d44aa6133126ac22094",
+       "layout": "IPY_MODEL_e7fe152978d2459c9cf82d8f9f257479",
        "placeholder": "​",
-       "style": "IPY_MODEL_1ed9a2e74124405fb3abc69c91bd1ea6",
+       "style": "IPY_MODEL_a6544264d0d741aa81b06d6cf1050e84",
        "tabbable": null,
        "tooltip": null,
-       "value": " 50/50 [00:02&lt;00:00, 22.98it/s]"
+       "value": " 50/50 [00:02&lt;00:00, 22.25it/s]"
       }
      },
-     "5df370f098ea41fba5fb462b17adbe25": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_75e910c8d71048ac9a0431c21969d577",
-        "IPY_MODEL_fee2c6fbc51f44eab84f3b42742d91e1",
-        "IPY_MODEL_1510c65a28504ad5b352dcd079536344"
-       ],
-       "layout": "IPY_MODEL_f6a821330d1046798295f553b4e852c9",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "5ef4034cdc7c4b43a254cd124d41d7ab": {
+     "3a73a2ce3d684e57af5ead4d970bb546": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -4417,15 +4364,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_34b16debba6742959118694ecadd4e80",
+       "layout": "IPY_MODEL_19da29c1b3cf4fae9710cd70f0baf271",
        "placeholder": "​",
-       "style": "IPY_MODEL_b224d7466bda47f88fa7756be3f08ef5",
+       "style": "IPY_MODEL_30e033f4a4a745f99416fa6c47e6b1b8",
        "tabbable": null,
        "tooltip": null,
        "value": "generation_config.json: 100%"
       }
      },
-     "6114d823a31a44f286d95a2e9ce2bf33": {
+     "3adaa7786f684ff6abb501929b91bfeb": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4478,109 +4425,80 @@
        "width": null
       }
      },
-     "68523653d5074bb6870277f003d5d605": {
+     "3b38cd8e7a4749a49d9442a541dd6d69": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_f38841b822954662b43dba4c39e860e5",
-       "max": 1355256.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_9069a6ad426248c09b77ea95dd5aee4a",
+       "layout": "IPY_MODEL_355529cac414494c83e338e54c92a4cc",
+       "placeholder": "​",
+       "style": "IPY_MODEL_72e1c6506989403db938acb8a05caa9a",
        "tabbable": null,
        "tooltip": null,
-       "value": 1355256.0
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 17.5MB/s]"
       }
      },
-     "6c874564389b4cc08af3017cb4823571": {
-      "model_module": "@jupyter-widgets/base",
+     "4025e90abd624e92ad601372fe03f72a": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "6e868c8f6c0448e7b61f10235b8c78a3": {
+     "4045d70e18ef41f892b496c2457200e8": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_7edd3fced4af4b77a0ddce7023ab8b7b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_7efc08a62c6f40298d362be028692ce6",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 17.6MB/s]"
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "7164ecd232594c6d854b1365370e4223": {
+     "40bb01d6e86d499a80c048f392bcbe29": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "4116100d429b4b9faf99f919b8cba2fd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -4596,93 +4514,61 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_a82d2103fcf049959a85dad02d097310",
-       "max": 456318.0,
+       "layout": "IPY_MODEL_ce2ad39062644901b02faaefcf98b910",
+       "max": 352824413.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_bae937ca0f53411b921ad90d6d430654",
+       "style": "IPY_MODEL_16df306b166340878aae8ee0d7d2af94",
        "tabbable": null,
        "tooltip": null,
-       "value": 456318.0
+       "value": 352824413.0
       }
      },
-     "718432154ef24d44aa6133126ac22094": {
-      "model_module": "@jupyter-widgets/base",
+     "45071b25ff23496094a139022c8edc54": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "FloatProgressModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_6e974e0f6e104aee841709766cbfe512",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_4025e90abd624e92ad601372fe03f72a",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 124.0
       }
      },
-     "75e910c8d71048ac9a0431c21969d577": {
+     "459712eff9964201b95e3cb31857e8da": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_f4944bf0e2044a9bb30a6e728cf9a4b0",
-       "placeholder": "​",
-       "style": "IPY_MODEL_fcee2cda87b34b6daaaa18c4471f41f0",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer_config.json: 100%"
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "79d408b7c64b44d0922d0edcc376a946": {
+     "46833033a8eb479ea4c5c87af7d997bb": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4735,7 +4621,7 @@
        "width": null
       }
      },
-     "7cb2aaf5bf784a5f952b312cc762a687": {
+     "47b00b0a700a470cba9e0970646bf5a3": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4788,7 +4674,7 @@
        "width": null
       }
      },
-     "7edd3fced4af4b77a0ddce7023ab8b7b": {
+     "485b5446c8864360a4f063c6992e772a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4841,7 +4727,56 @@
        "width": null
       }
      },
-     "7efc08a62c6f40298d362be028692ce6": {
+     "4d16a5f6738f41918a9be1a5c6911d80": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_c6ee4432596b4ff999fe1bddd0d46240",
+       "placeholder": "​",
+       "style": "IPY_MODEL_697afa5733c6440f907ee2c1ffb2584e",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 456k/456k [00:00&lt;00:00, 35.6MB/s]"
+      }
+     },
+     "4ec86ba49fc04f1ea410f2a37bf7ffae": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_b49e96fd18bc45e981e58cefbd348841",
+       "max": 456318.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_4045d70e18ef41f892b496c2457200e8",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 456318.0
+      }
+     },
+     "4f9bedf73f97497db82c1b58ff5117bf": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4859,7 +4794,23 @@
        "text_color": null
       }
      },
-     "81593b1d4ed44438ac88b2d7a8b8cb96": {
+     "546e22db36d04372a0aa187adbcf0262": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "58a2ce5740e543bdaa92d598b1b084dc": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "ProgressStyleModel",
@@ -4875,7 +4826,7 @@
        "description_width": ""
       }
      },
-     "83789bfd75804ec290640f71987c3492": {
+     "5ad68d8e26e44e76a786ef4b57a2263d": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4893,31 +4844,92 @@
        "text_color": null
       }
      },
-     "8886699251a949f49e757b9fdc7ca890": {
+     "5fbc42bb13f64054a6dbbd8c58ecfc59": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "6304f591778f49058f9403f77c821c56": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_181e4adfc3bc45cd965a56602246337a",
-        "IPY_MODEL_1c96e44af3b24a34aaff4045e496a27d",
-        "IPY_MODEL_27310f32e9974dcd94aa2b9c49c1e6e7"
-       ],
-       "layout": "IPY_MODEL_6c874564389b4cc08af3017cb4823571",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_aa07cd1a18be4feda9dfaae9577809b5",
+       "placeholder": "​",
+       "style": "IPY_MODEL_5fbc42bb13f64054a6dbbd8c58ecfc59",
        "tabbable": null,
-       "tooltip": null
+       "tooltip": null,
+       "value": " 353M/353M [00:01&lt;00:00, 198MB/s]"
+      }
+     },
+     "697afa5733c6440f907ee2c1ffb2584e": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "69da672452a94ea68b438b3be68f1a24": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_0347d987e2964dd4a31348434040c0c7",
+       "max": 50.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_858356c58e784bc4b5778ed67747a0d1",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 50.0
       }
      },
-     "8aa3d272612246cb9ffad05643ac4d7d": {
+     "6e974e0f6e104aee841709766cbfe512": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4970,65 +4982,30 @@
        "width": null
       }
      },
-     "8d0044c9d9594cdc99a744683335c85f": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "8efd0c3ea3bc4e5d8c1b966bff063c3a": {
+     "70759237fca04d3084c2a5df2a30f271": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_5ef4034cdc7c4b43a254cd124d41d7ab",
-        "IPY_MODEL_397d196b08ea47da94f5d96dd6276340",
-        "IPY_MODEL_c8831f07d5264cae9518fd997bd1464f"
-       ],
-       "layout": "IPY_MODEL_c277db05726a40f691e0fec1426c30e2",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_47b00b0a700a470cba9e0970646bf5a3",
+       "placeholder": "​",
+       "style": "IPY_MODEL_459712eff9964201b95e3cb31857e8da",
        "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "9069a6ad426248c09b77ea95dd5aee4a": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "tooltip": null,
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 9.38MB/s]"
       }
      },
-     "9bd1760890dd4e298154efe56e5a757d": {
+     "7129d439a9d549d48802b08586b8c24f": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5081,7 +5058,25 @@
        "width": null
       }
      },
-     "9d118b82362542f59e0227f8f93cf219": {
+     "72e1c6506989403db938acb8a05caa9a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "76995f1052794da19381d92e27c1de96": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5134,7 +5129,7 @@
        "width": null
       }
      },
-     "a452f9720b0141f18f4f1585cfbcf102": {
+     "7f7ad53041c6499e9178b752e3f0bf66": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5187,7 +5182,30 @@
        "width": null
       }
      },
-     "a678c766e93d424daa0150a52a8f38a2": {
+     "8480a1acfe1a4929becf5c1eeb2df5f1": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_e4da68381e90404db807b341d42c96a8",
+       "placeholder": "​",
+       "style": "IPY_MODEL_4f9bedf73f97497db82c1b58ff5117bf",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "100%"
+      }
+     },
+     "85239cb8d6ba4d288b001290e919d816": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -5202,16 +5220,107 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_17bb4044cad04889b7a3185f7cca5d61",
-        "IPY_MODEL_7164ecd232594c6d854b1365370e4223",
-        "IPY_MODEL_2eb8521bbd3d441180ebfef6c4e91184"
+        "IPY_MODEL_0f1f32e4ccb44e77b4a1aab830f601a5",
+        "IPY_MODEL_db49cd92e84f4f20b4d44fe5829790df",
+        "IPY_MODEL_70759237fca04d3084c2a5df2a30f271"
        ],
-       "layout": "IPY_MODEL_79d408b7c64b44d0922d0edcc376a946",
+       "layout": "IPY_MODEL_c666ac6d4533412e922fc4b9a1084fc1",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "a82d2103fcf049959a85dad02d097310": {
+     "858356c58e784bc4b5778ed67747a0d1": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "975c4858e139485f8f0aa84771e564a4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_25f38d8b724d481397c21ffd4e2ae408",
+       "max": 1042301.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_546e22db36d04372a0aa187adbcf0262",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1042301.0
+      }
+     },
+     "977babee536f42158a1d81b4b7aa208d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_2974465a897b4ac9ba7d573f8e1ea5f8",
+       "max": 12.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_58a2ce5740e543bdaa92d598b1b084dc",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 12.0
+      }
+     },
+     "98db25d1775c4574b9702ae9d27f9d76": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_bc420542d1d34e14978dcc6d9d0471e3",
+       "placeholder": "​",
+       "style": "IPY_MODEL_c741d615fd45485b899d67710062ed85",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "vocab.json: 100%"
+      }
+     },
+     "9fce92e16f574800b2a9add4b6c10ef8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5264,7 +5373,23 @@
        "width": null
       }
      },
-     "af4c8704f5ac48e18155eafc53cb00cf": {
+     "a05caff4cdcb42e8813ccaf3072ea092": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "a3a46ce06ec04834810c4ab96c959684": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5279,15 +5404,33 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_1fa06c806c4a4980bbe87f0746c9a335",
+       "layout": "IPY_MODEL_a95e7f64a7d240529cacf7a5a9e1c06b",
        "placeholder": "​",
-       "style": "IPY_MODEL_d8ec5600d09e4881a86bbf234f64eee7",
+       "style": "IPY_MODEL_40bb01d6e86d499a80c048f392bcbe29",
        "tabbable": null,
        "tooltip": null,
-       "value": "tokenizer.json: 100%"
+       "value": " 26.0/26.0 [00:00&lt;00:00, 5.03kB/s]"
+      }
+     },
+     "a3feec14402a46afa537597c59f70647": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "b224d7466bda47f88fa7756be3f08ef5": {
+     "a6544264d0d741aa81b06d6cf1050e84": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5305,7 +5448,7 @@
        "text_color": null
       }
      },
-     "b43f9e4d856841f4a148132df957a6bb": {
+     "a6d329232ced49958c43f07f02862115": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5358,25 +5501,7 @@
        "width": null
       }
      },
-     "b5b250cb0b404e5dbfb096747062a997": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "b5cfa8bab51f4768968002c31b9c4ccc": {
+     "a95e7f64a7d240529cacf7a5a9e1c06b": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5429,23 +5554,7 @@
        "width": null
       }
      },
-     "bae937ca0f53411b921ad90d6d430654": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "c277db05726a40f691e0fec1426c30e2": {
+     "aa07cd1a18be4feda9dfaae9577809b5": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5498,130 +5607,64 @@
        "width": null
       }
      },
-     "c4fc9d6b249f46218d1b6b1088d1852f": {
+     "ab0021bee3b94de0965c25c38a7962f2": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_b43f9e4d856841f4a148132df957a6bb",
-       "placeholder": "​",
-       "style": "IPY_MODEL_e36e177cd6b047a6a4ef203d26f44bb3",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 14.5MB/s]"
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "c8831f07d5264cae9518fd997bd1464f": {
+     "ae9c1384a8ef40a9b2a32844f1646716": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_db618c1d50364bed9940e58ee948cedf",
-       "placeholder": "​",
-       "style": "IPY_MODEL_5054bc9f6dc94e5a938d3b2a410a235f",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 22.2kB/s]"
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "ca0421e9c3074fe38697b4e54fe160b3": {
+     "aec5d70e86d84aef8fe50515973443a8": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_099201fc3592463a99cefa3adf51b80e",
-        "IPY_MODEL_0813474f79c445e4969ff3384cba2b31",
-        "IPY_MODEL_5adce76b5f7845b087083c2207ab56d3"
-       ],
-       "layout": "IPY_MODEL_6114d823a31a44f286d95a2e9ce2bf33",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_46833033a8eb479ea4c5c87af7d997bb",
+       "placeholder": "​",
+       "style": "IPY_MODEL_5ad68d8e26e44e76a786ef4b57a2263d",
        "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "d027f2bb85564a00bf60cfe08b9b71ff": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "tooltip": null,
+       "value": "merges.txt: 100%"
       }
      },
-     "d23203daf6b546699057c5635a148bad": {
+     "b49e96fd18bc45e981e58cefbd348841": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5674,7 +5717,7 @@
        "width": null
       }
      },
-     "d5f0c8842a284fdda4dfa3206605c35f": {
+     "baa802cac1bf4a77bd2d6f2da4e0c206": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -5689,16 +5732,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_f88e08bf6447493e9ee596e52053d053",
-        "IPY_MODEL_483d3536ea8644e0932ff376a08a7509",
-        "IPY_MODEL_c4fc9d6b249f46218d1b6b1088d1852f"
+        "IPY_MODEL_aec5d70e86d84aef8fe50515973443a8",
+        "IPY_MODEL_4ec86ba49fc04f1ea410f2a37bf7ffae",
+        "IPY_MODEL_4d16a5f6738f41918a9be1a5c6911d80"
        ],
-       "layout": "IPY_MODEL_d7e889b34a51418193c95f469bf95c6d",
+       "layout": "IPY_MODEL_485b5446c8864360a4f063c6992e772a",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "d7e889b34a51418193c95f469bf95c6d": {
+     "bc420542d1d34e14978dcc6d9d0471e3": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5751,7 +5794,25 @@
        "width": null
       }
      },
-     "d8ec5600d09e4881a86bbf234f64eee7": {
+     "bd751bbba24c4154acc75b88e0f3a87a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "bfdbeaa99bc942abb6a71e94629483ab": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5769,7 +5830,7 @@
        "text_color": null
       }
      },
-     "d99e129cdb3147aa913c3c969c4c5910": {
+     "c666ac6d4533412e922fc4b9a1084fc1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5822,31 +5883,7 @@
        "width": null
       }
      },
-     "da7af55f6bb5479b852a41df346f73ee": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_af4c8704f5ac48e18155eafc53cb00cf",
-        "IPY_MODEL_68523653d5074bb6870277f003d5d605",
-        "IPY_MODEL_6e868c8f6c0448e7b61f10235b8c78a3"
-       ],
-       "layout": "IPY_MODEL_7cb2aaf5bf784a5f952b312cc762a687",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "db618c1d50364bed9940e58ee948cedf": {
+     "c6ee4432596b4ff999fe1bddd0d46240": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5899,7 +5936,7 @@
        "width": null
       }
      },
-     "df6f557d81e94e78b53b3b8791c846c4": {
+     "c70847ad8e5543748ffbab29446b51c1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5914,51 +5951,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_ec306a831c0c4f55b9058110a79d8978",
+       "layout": "IPY_MODEL_7f7ad53041c6499e9178b752e3f0bf66",
        "placeholder": "​",
-       "style": "IPY_MODEL_83789bfd75804ec290640f71987c3492",
+       "style": "IPY_MODEL_a3feec14402a46afa537597c59f70647",
        "tabbable": null,
        "tooltip": null,
        "value": "model.safetensors: 100%"
       }
      },
-     "e36e177cd6b047a6a4ef203d26f44bb3": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "e3d82c176b4b452e93686bf9f1c9d3e1": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "e75a1f080c744b1c8939f0abb8e708fc": {
+     "c741d615fd45485b899d67710062ed85": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5976,30 +5977,7 @@
        "text_color": null
       }
      },
-     "e8077a539c2f4fe7a0bc96e4cd70fa7e": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_1f2d34420a9d422db4c26669c9eece1b",
-       "placeholder": "​",
-       "style": "IPY_MODEL_e3d82c176b4b452e93686bf9f1c9d3e1",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 353M/353M [00:01&lt;00:00, 248MB/s]"
-      }
-     },
-     "ec306a831c0c4f55b9058110a79d8978": {
+     "ce2ad39062644901b02faaefcf98b910": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6052,7 +6030,30 @@
        "width": null
       }
      },
-     "ed792737675641488478dfe9e706af53": {
+     "d671580b566f41a3b09b9e31d0a97129": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_f825f61e07764da39007e09a988d2df0",
+       "placeholder": "​",
+       "style": "IPY_MODEL_dee4a33c12ba4787ad99f56355b782af",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer_config.json: 100%"
+      }
+     },
+     "db49cd92e84f4f20b4d44fe5829790df": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -6068,17 +6069,59 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_2ce16701f86e465fa67e824fed52d560",
-       "max": 352824413.0,
+       "layout": "IPY_MODEL_f5bf41cb59ed4db4b184f40d99bfbc1a",
+       "max": 1355256.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_fb7967775a7c426ab326120208b1c1d0",
+       "style": "IPY_MODEL_ab0021bee3b94de0965c25c38a7962f2",
        "tabbable": null,
        "tooltip": null,
-       "value": 352824413.0
+       "value": 1355256.0
+      }
+     },
+     "ddd91d21a45e49a8a90fd6693fc0fe90": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_8480a1acfe1a4929becf5c1eeb2df5f1",
+        "IPY_MODEL_977babee536f42158a1d81b4b7aa208d",
+        "IPY_MODEL_fdf5d6542de342d08a2d9a0482eda1d7"
+       ],
+       "layout": "IPY_MODEL_76995f1052794da19381d92e27c1de96",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "dee4a33c12ba4787ad99f56355b782af": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "f38841b822954662b43dba4c39e860e5": {
+     "e4da68381e90404db807b341d42c96a8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6131,7 +6174,7 @@
        "width": null
       }
      },
-     "f4944bf0e2044a9bb30a6e728cf9a4b0": {
+     "e7fe152978d2459c9cf82d8f9f257479": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6184,7 +6227,57 @@
        "width": null
       }
      },
-     "f6a821330d1046798295f553b4e852c9": {
+     "ef67630185584553867488aa7abe9fa9": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_1dfb46a04fe14273b07f0621a7a25096",
+        "IPY_MODEL_69da672452a94ea68b438b3be68f1a24",
+        "IPY_MODEL_38bdea7d9b64452ea8420690243965cf"
+       ],
+       "layout": "IPY_MODEL_217459b8733641fda6a21621dc147583",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "f50d3b1ce11a44d8b5801bdd32a86fa8": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_3383a9989d6a4375bccecc436f7df6c0",
+       "max": 26.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_a05caff4cdcb42e8813ccaf3072ea092",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 26.0
+      }
+     },
+     "f5bf41cb59ed4db4b184f40d99bfbc1a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6237,46 +6330,7 @@
        "width": null
       }
      },
-     "f88e08bf6447493e9ee596e52053d053": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_12c5e2e95f084803834b82ff36d5a9aa",
-       "placeholder": "​",
-       "style": "IPY_MODEL_b5b250cb0b404e5dbfb096747062a997",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "vocab.json: 100%"
-      }
-     },
-     "fb7967775a7c426ab326120208b1c1d0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "fcc3cac4665349559e159dd7b1be5c7f": {
+     "f825f61e07764da39007e09a988d2df0": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6329,48 +6383,27 @@
        "width": null
       }
      },
-     "fcee2cda87b34b6daaaa18c4471f41f0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "fee2c6fbc51f44eab84f3b42742d91e1": {
+     "fdf5d6542de342d08a2d9a0482eda1d7": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_40f00596573845cfb4f6ee3bf57fcf62",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_1e61511b645b4c07aabc82cde43f0707",
+       "layout": "IPY_MODEL_22fa2830a0ab4ae4bb190db8ab08f59c",
+       "placeholder": "​",
+       "style": "IPY_MODEL_bd751bbba24c4154acc75b88e0f3a87a",
        "tabbable": null,
        "tooltip": null,
-       "value": 26.0
+       "value": " 12/12 [00:19&lt;00:00,  1.65s/it]"
       }
      }
     },
diff --git a/generated/model_properties_table.html b/generated/model_properties_table.html
index 18315f982..692d87b52 100644
--- a/generated/model_properties_table.html
+++ b/generated/model_properties_table.html
@@ -2331,7 +2331,187 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>18944</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>phi-1</p></td>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-0.5B</p></td>
+<td class="text-left"><p>391M</p></td>
+<td class="text-right"><p>24</p></td>
+<td class="text-right"><p>896</p></td>
+<td class="text-right"><p>14</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>4864</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-0.5B-Instruct</p></td>
+<td class="text-left"><p>391M</p></td>
+<td class="text-right"><p>24</p></td>
+<td class="text-right"><p>896</p></td>
+<td class="text-right"><p>14</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>4864</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-1.5B</p></td>
+<td class="text-left"><p>1.4B</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-right"><p>1536</p></td>
+<td class="text-right"><p>12</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>8960</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-1.5B-Instruct</p></td>
+<td class="text-left"><p>1.4B</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-right"><p>1536</p></td>
+<td class="text-right"><p>12</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>8960</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-3B</p></td>
+<td class="text-left"><p>3.0B</p></td>
+<td class="text-right"><p>36</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>16</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>11008</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-3B-Instruct</p></td>
+<td class="text-left"><p>3.0B</p></td>
+<td class="text-right"><p>36</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>16</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>151936</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>11008</p></td>
+<td class="text-left"><p>2</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-7B</p></td>
+<td class="text-left"><p>7.1B</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-right"><p>3584</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>18944</p></td>
+<td class="text-left"><p>4</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-7B-Instruct</p></td>
+<td class="text-left"><p>7.1B</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-right"><p>3584</p></td>
+<td class="text-right"><p>28</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>18944</p></td>
+<td class="text-left"><p>4</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-14B</p></td>
+<td class="text-left"><p>15B</p></td>
+<td class="text-right"><p>48</p></td>
+<td class="text-right"><p>5120</p></td>
+<td class="text-right"><p>40</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>13824</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-14B-Instruct</p></td>
+<td class="text-left"><p>15B</p></td>
+<td class="text-right"><p>48</p></td>
+<td class="text-right"><p>5120</p></td>
+<td class="text-right"><p>40</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>13824</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-32B</p></td>
+<td class="text-left"><p>34B</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>5120</p></td>
+<td class="text-right"><p>40</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>27648</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-32B-Instruct</p></td>
+<td class="text-left"><p>34B</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>5120</p></td>
+<td class="text-right"><p>40</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>27648</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/Qwen2.5-72B</p></td>
+<td class="text-left"><p>80B</p></td>
+<td class="text-right"><p>80</p></td>
+<td class="text-right"><p>8192</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>29568</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>Qwen/Qwen2.5-72B-Instruct</p></td>
+<td class="text-left"><p>80B</p></td>
+<td class="text-right"><p>80</p></td>
+<td class="text-right"><p>8192</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>29568</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-even"><td class="text-left"><p>Qwen/QwQ-32B-Preview</p></td>
+<td class="text-left"><p>34B</p></td>
+<td class="text-right"><p>64</p></td>
+<td class="text-right"><p>5120</p></td>
+<td class="text-right"><p>40</p></td>
+<td class="text-left"><p>silu</p></td>
+<td class="text-right"><p>2048</p></td>
+<td class="text-right"><p>152064</p></td>
+<td class="text-right"><p>128</p></td>
+<td class="text-right"><p>27648</p></td>
+<td class="text-left"><p>8</p></td>
+</tr>
+<tr class="row-odd"><td class="text-left"><p>phi-1</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2343,7 +2523,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>phi-1_5</p></td>
+<tr class="row-even"><td class="text-left"><p>phi-1_5</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2355,7 +2535,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>phi-2</p></td>
+<tr class="row-odd"><td class="text-left"><p>phi-2</p></td>
 <td class="text-left"><p>2.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>2560</p></td>
@@ -2367,7 +2547,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>10240</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>phi-3</p></td>
+<tr class="row-even"><td class="text-left"><p>phi-3</p></td>
 <td class="text-left"><p>3.6B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2379,7 +2559,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>8192</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2b</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2b</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>18</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2391,7 +2571,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p>1</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-7b</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-7b</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2403,7 +2583,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2b-it</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2b-it</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>18</p></td>
 <td class="text-right"><p>2048</p></td>
@@ -2415,7 +2595,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>16384</p></td>
 <td class="text-left"><p>1</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-7b-it</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-7b-it</p></td>
 <td class="text-left"><p>7.8B</p></td>
 <td class="text-right"><p>28</p></td>
 <td class="text-right"><p>3072</p></td>
@@ -2427,7 +2607,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>24576</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-2b</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-2b</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>26</p></td>
 <td class="text-right"><p>2304</p></td>
@@ -2439,7 +2619,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>9216</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-2b-it</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-2b-it</p></td>
 <td class="text-left"><p>2.1B</p></td>
 <td class="text-right"><p>26</p></td>
 <td class="text-right"><p>2304</p></td>
@@ -2451,7 +2631,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>9216</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-9b</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-9b</p></td>
 <td class="text-left"><p>8.9B</p></td>
 <td class="text-right"><p>42</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2463,7 +2643,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-9b-it</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-9b-it</p></td>
 <td class="text-left"><p>8.9B</p></td>
 <td class="text-right"><p>42</p></td>
 <td class="text-right"><p>3584</p></td>
@@ -2475,7 +2655,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>14336</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>gemma-2-27b</p></td>
+<tr class="row-odd"><td class="text-left"><p>gemma-2-27b</p></td>
 <td class="text-left"><p>27B</p></td>
 <td class="text-right"><p>46</p></td>
 <td class="text-right"><p>4608</p></td>
@@ -2487,7 +2667,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>36864</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>gemma-2-27b-it</p></td>
+<tr class="row-even"><td class="text-left"><p>gemma-2-27b-it</p></td>
 <td class="text-left"><p>27B</p></td>
 <td class="text-right"><p>46</p></td>
 <td class="text-right"><p>4608</p></td>
@@ -2499,7 +2679,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>36864</p></td>
 <td class="text-left"><p>16</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>yi-6b</p></td>
+<tr class="row-odd"><td class="text-left"><p>yi-6b</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2511,7 +2691,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>yi-34b</p></td>
+<tr class="row-even"><td class="text-left"><p>yi-34b</p></td>
 <td class="text-left"><p>39B</p></td>
 <td class="text-right"><p>60</p></td>
 <td class="text-right"><p>7168</p></td>
@@ -2523,7 +2703,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>20480</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>yi-6b-chat</p></td>
+<tr class="row-odd"><td class="text-left"><p>yi-6b-chat</p></td>
 <td class="text-left"><p>6.5B</p></td>
 <td class="text-right"><p>32</p></td>
 <td class="text-right"><p>4096</p></td>
@@ -2535,7 +2715,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>11008</p></td>
 <td class="text-left"><p>4</p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>yi-34b-chat</p></td>
+<tr class="row-even"><td class="text-left"><p>yi-34b-chat</p></td>
 <td class="text-left"><p>39B</p></td>
 <td class="text-right"><p>60</p></td>
 <td class="text-right"><p>7168</p></td>
@@ -2547,7 +2727,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>20480</p></td>
 <td class="text-left"><p>8</p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>t5-small</p></td>
+<tr class="row-odd"><td class="text-left"><p>t5-small</p></td>
 <td class="text-left"><p>19M</p></td>
 <td class="text-right"><p>6</p></td>
 <td class="text-right"><p>512</p></td>
@@ -2559,7 +2739,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>2048</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>t5-base</p></td>
+<tr class="row-even"><td class="text-left"><p>t5-base</p></td>
 <td class="text-left"><p>85M</p></td>
 <td class="text-right"><p>12</p></td>
 <td class="text-right"><p>768</p></td>
@@ -2571,7 +2751,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>3072</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-even"><td class="text-left"><p>t5-large</p></td>
+<tr class="row-odd"><td class="text-left"><p>t5-large</p></td>
 <td class="text-left"><p>302M</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>1024</p></td>
@@ -2583,7 +2763,7 @@ <h1>Model Properties Table<a class="headerlink" href="#model-properties-table" t
 <td class="text-right"><p>4096</p></td>
 <td class="text-left"><p></p></td>
 </tr>
-<tr class="row-odd"><td class="text-left"><p>mGPT</p></td>
+<tr class="row-even"><td class="text-left"><p>mGPT</p></td>
 <td class="text-left"><p>1.2B</p></td>
 <td class="text-right"><p>24</p></td>
 <td class="text-right"><p>2048</p></td>
diff --git a/searchindex.js b/searchindex.js
index 1ebf21655..18feeca65 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 14, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3, 73], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7, 73], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70, 73], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71, 73], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 14, 15, 38, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 38, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 15, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71, 73], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 15, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71, 73], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 14, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72, 73], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 69, 70, 71, 73], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71, 73], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 15, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71, 73], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 15, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 69, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38, 73], "through": [5, 14, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71, 73], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71, 73], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71, 73], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": 5, "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 15, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 14, 18, 38, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 15, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 71], "fix": [5, 37, 70, 71], "exist": [5, 14, 36, 38, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 14, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 69, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71, 73], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71, 73], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 38, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 14, 18, 19, 24, 30, 37, 38, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 15, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70, 73], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": 14, "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": 14, "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "first_n_lay": [14, 38], "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "resolut": [14, 38], "cfg_dict": [14, 38], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 15, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_ungroup_grouped_query_attent": 14, "ungroup_grouped_query_attent": [14, 15], "ungroup": [14, 15], "gqa": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "use_ntk_by_parts_rop": 15, "ntk_by_parts_low_freq_factor": 15, "ntk_by_parts_high_freq_factor": 15, "ntk_by_parts_factor": 15, "8": [15, 18, 35, 36, 70, 71, 72], "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "ntk": 15, "adjust": [15, 70, 71], "interpol": 15, "2309": 15, "00071": 15, "threshold": 15, "high": [15, 69, 70, 71], "rate": [15, 63, 71], "strategi": [15, 71], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": 36, "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "nemo": [38, 72], "2407": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": [70, 73], "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7fcdda667510": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": 71, "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "43": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "assert": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "53": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "780": 71, "784": 71, "56": [71, 72], "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "58": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "32b": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "131072": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73, "courtesi": 73, "austin": 73, "kozlowski": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_ungroup_grouped_query_attention"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "NTK_by_parts_factor"], [15, 4, 1, "", "NTK_by_parts_high_freq_factor"], [15, 4, 1, "", "NTK_by_parts_low_freq_factor"], [15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 4, 1, "", "ungroup_grouped_query_attention"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_NTK_by_parts_rope"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_ungroup_grouped_query_attention() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_ungroup_grouped_query_attention"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "ntk_by_parts_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_factor"]], "ntk_by_parts_high_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_high_freq_factor"]], "ntk_by_parts_low_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_low_freq_factor"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "ungroup_grouped_query_attention (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.ungroup_grouped_query_attention"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_ntk_by_parts_rope (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_NTK_by_parts_rope"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 14, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3, 73], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7, 73], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70, 73], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71, 73], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 71, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 37, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 37, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 14, 15, 38, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 38, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 15, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71, 73], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 15, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71, 73], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 14, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72, 73], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 37, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 69, 70, 71, 73], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71, 73], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69, 71], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 37, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 15, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71, 73], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 15, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 69, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38, 73], "through": [5, 14, 37, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71, 73], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 37, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71, 73], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71, 73], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": [5, 71], "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 15, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 14, 18, 38, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70, 71], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 15, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 71], "fix": [5, 37, 70, 71], "exist": [5, 14, 36, 38, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 14, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 69, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70, 71], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71, 73], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71, 73], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 38, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15, 71], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 14, 18, 19, 24, 30, 37, 38, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11, 71], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 15, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70, 73], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": [14, 71], "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": [14, 71], "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "first_n_lay": [14, 38], "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "resolut": [14, 38], "cfg_dict": [14, 38], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 15, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70, 71], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_ungroup_grouped_query_attent": 14, "ungroup_grouped_query_attent": [14, 15], "ungroup": [14, 15], "gqa": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "use_ntk_by_parts_rop": 15, "ntk_by_parts_low_freq_factor": 15, "ntk_by_parts_high_freq_factor": 15, "ntk_by_parts_factor": 15, "8": [15, 18, 35, 36, 70, 71, 72], "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "ntk": 15, "adjust": [15, 70, 71], "interpol": 15, "2309": 15, "00071": 15, "threshold": 15, "high": [15, 69, 70, 71], "rate": [15, 63, 71], "strategi": [15, 71], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": [36, 71], "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70, 71], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "nemo": [38, 72], "2407": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "32b": [38, 72], "72b": [38, 72], "qwq": [38, 72], "preview": [38, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 71, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": [70, 73], "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7f0f50f9df90": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "cell": 71, "attn_hook_nam": 71, "attn_lay": 71, "gpt2_attn_cach": 71, "gpt2_attn": 71, "assert": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": [71, 72], "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "43": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "53": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "56": [71, 72], "780": 71, "784": 71, "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "58": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "131072": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "27648": 72, "80b": 72, "29568": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73, "courtesi": 73, "austin": 73, "kozlowski": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_ungroup_grouped_query_attention"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "NTK_by_parts_factor"], [15, 4, 1, "", "NTK_by_parts_high_freq_factor"], [15, 4, 1, "", "NTK_by_parts_low_freq_factor"], [15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 4, 1, "", "ungroup_grouped_query_attention"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_NTK_by_parts_rope"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_ungroup_grouped_query_attention() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_ungroup_grouped_query_attention"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "ntk_by_parts_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_factor"]], "ntk_by_parts_high_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_high_freq_factor"]], "ntk_by_parts_low_freq_factor (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.NTK_by_parts_low_freq_factor"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "ungroup_grouped_query_attention (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.ungroup_grouped_query_attention"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_ntk_by_parts_rope (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_NTK_by_parts_rope"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file